From 9bdb94bff0100850d9ac95afcb81b34f19d9a4b6 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 23 Sep 2019 19:23:18 +0100 Subject: [PATCH 01/49] wip move middleware to seperate package --- app.go | 50 ++- auth/auth.go | 59 ++- auth/auth_test.go | 70 ++-- auth/chainconfig_middleware.go | 29 +- auth/chainconfig_middleware_test.go | 35 +- auth/{ => keys}/config.go | 2 +- auth/keys/keys.go | 43 +++ auth/multi_chain_sigtx_middleware.go | 61 ++-- auth/multi_chain_sigtx_middleware_test.go | 20 +- sequence.go => auth/sequence/sequence.go | 2 +- cmd/loom/loom.go | 339 ++++++++++-------- config/config.go | 35 +- evm/config.go => config/evm_config.go | 2 +- config/throttle_config.go | 82 +++++ eth/polls/polls_test.go | 16 +- eth/query/block.go | 3 +- eth/query/query_test.go | 19 +- events/db_indexer.go | 4 +- events/log.go | 5 +- events/redis.go | 5 +- evm/loomevm.go | 4 +- fork.go | 9 +- plugin/vm_test.go | 4 +- receipts/common/common.go | 17 +- receipts/handler/handler.go | 5 - receipts/handler/handler_test.go | 5 +- rpc/debug/jsonrpc_conversion.go | 29 ++ rpc/debug/trace.go | 35 ++ rpc/instrumenting.go | 14 + rpc/mock_query_server.go | 8 + rpc/query_server.go | 23 +- rpc/query_server_test.go | 12 + rpc/query_service.go | 7 + store/splitstore.go | 78 ++++ throttle/contract_tx_limiter_middleware.go | 240 ------------- .../contract_tx_limiter_middleware_test.go | 6 +- throttle/karma-middleware.go | 20 +- throttle/legacy_deployer_middleware.go | 46 +-- throttle/middleware_test_helper.go | 14 +- throttle/throttle.go | 6 +- throttle/throttle_test.go | 4 +- throttle/tx_limiter_middleware.go | 93 ----- tx_handler/migration_tx_handler.go | 15 +- middleware.go => txhandler/middleware.go | 2 +- .../middleware}/deployer-middleware.go | 43 ++- .../middleware}/deployer-middleware_test.go | 8 +- txhandler/middleware/memory_app.go | 80 +++++ router.go => txhandler/middleware/router.go | 21 +- txhandler/middleware/throttle.go | 276 ++++++++++++++ .../middleware_test.go | 2 +- txhandler/txhandler.go | 27 ++ vm/handler.go | 16 +- 52 files changed, 1215 insertions(+), 835 deletions(-) rename auth/{ => keys}/config.go (98%) create mode 100644 auth/keys/keys.go rename sequence.go => auth/sequence/sequence.go (97%) rename evm/config.go => config/evm_config.go (98%) create mode 100644 config/throttle_config.go create mode 100644 rpc/debug/jsonrpc_conversion.go create mode 100644 rpc/debug/trace.go create mode 100644 store/splitstore.go rename middleware.go => txhandler/middleware.go (99%) rename {throttle => txhandler/middleware}/deployer-middleware.go (86%) rename {throttle => txhandler/middleware}/deployer-middleware_test.go (91%) create mode 100644 txhandler/middleware/memory_app.go rename router.go => txhandler/middleware/router.go (76%) create mode 100644 txhandler/middleware/throttle.go rename middleware_test.go => txhandler/middleware_test.go (99%) create mode 100644 txhandler/txhandler.go diff --git a/app.go b/app.go index 9d31bccfbd..2e05308875 100644 --- a/app.go +++ b/app.go @@ -6,18 +6,19 @@ import ( "fmt" "time" - "github.com/loomnetwork/loomchain/eth/utils" - "github.com/loomnetwork/loomchain/features" - "github.com/loomnetwork/loomchain/registry" - "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" stdprometheus "github.com/prometheus/client_golang/prometheus" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/common" ttypes "github.com/tendermint/tendermint/types" + "github.com/loomnetwork/loomchain/eth/utils" + "github.com/loomnetwork/loomchain/features" + "github.com/loomnetwork/loomchain/registry" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/txhandler/middleware" + "github.com/loomnetwork/loomchain/log" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" @@ -25,25 +26,6 @@ import ( evmaux "github.com/loomnetwork/loomchain/store/evm_aux" ) -type TxHandler interface { - ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) -} - -type TxHandlerFunc func(state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) - -type TxHandlerResult struct { - Data []byte - ValidatorUpdates []abci.Validator - Info string - // Tags to associate with the tx that produced this result. Tags can be used to filter txs - // via the ABCI query interface (see https://godoc.org/github.com/tendermint/tendermint/libs/pubsub/query) - Tags []common.KVPair -} - -func (f TxHandlerFunc) ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { - return f(state, txBytes, isCheckTx) -} - type QueryHandler interface { Handle(state appstate.ReadOnlyState, path string, data []byte) ([]byte, error) } @@ -72,7 +54,7 @@ type Application struct { curBlockHash []byte Store store.VersionedKVStore Init func(appstate.State) error - TxHandler + txhandler.TxHandler QueryHandler EventHandler ReceiptHandlerProvider @@ -88,6 +70,7 @@ type Application struct { config *cctypes.Config childTxRefs []evmaux.ChildTxRef // links Tendermint txs to EVM txs ReceiptsVersion int32 + DebugTxHandler txhandler.TxHandler } var _ abci.Application = &Application{} @@ -437,7 +420,7 @@ func (a *Application) deliverTx(storeTx store.KVStoreTx, txBytes []byte) abci.Re return abci.ResponseDeliverTx{Code: abci.CodeTypeOK, Data: r.Data, Tags: r.Tags, Info: r.Info} } -func (a *Application) processTx(storeTx store.KVStoreTx, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { +func (a *Application) processTx(storeTx store.KVStoreTx, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { state := appstate.NewStoreState( context.Background(), storeTx, @@ -621,3 +604,18 @@ func (a *Application) ReadOnlyState() appstate.State { a.GetValidatorSet, ) } + +func (a *Application) InMemoryApp(blockNumber uint64) middleware.InMemoryApp { + // TODO: the store snapshot should be created atomically, otherwise the block header might + // not match the state... need to figure out why this hasn't spectacularly failed already + return middleware.NewInMemoryApp( + a.lastBlockHeader, + a.curBlockHeader, + a.curBlockHash, + a.Store, + a.TxHandler, + a.ReceiptsVersion, + a.GetValidatorSet, + a.config, + ) +} diff --git a/auth/auth.go b/auth/auth.go index 9b016ac22c..7f3a9da7b3 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -13,9 +13,11 @@ import ( loom "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/util" - "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/auth/keys" + "github.com/loomnetwork/loomchain/auth/sequence" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" ) var ( @@ -31,28 +33,13 @@ func init() { }, []string{}) } -type contextKey string - -func (c contextKey) String() string { - return "auth " + string(c) -} - -var ( - ContextKeyOrigin = contextKey("origin") - ContextKeyCheckTx = contextKey("CheckTx") -) - -func Origin(ctx context.Context) loom.Address { - return ctx.Value(ContextKeyOrigin).(loom.Address) -} - -var SignatureTxMiddleware = loomchain.TxMiddlewareFunc(func( +var SignatureTxMiddleware = txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult var tx SignedTx err := proto.Unmarshal(txBytes, &tx) @@ -65,7 +52,7 @@ var SignatureTxMiddleware = loomchain.TxMiddlewareFunc(func( return r, err } - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) return next(state.WithContext(ctx), tx.Inner, isCheckTx) }) @@ -93,7 +80,7 @@ func nonceKey(addr loom.Address) []byte { } func Nonce(state appstate.ReadOnlyState, addr loom.Address) uint64 { - return loomchain.NewSequence(nonceKey(addr)).Value(state) + return sequence.NewSequence(nonceKey(addr)).Value(state) } type NonceHandler struct { @@ -105,11 +92,11 @@ func (n *NonceHandler) Nonce( state appstate.State, kvStore store.KVStore, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult - origin := Origin(state.Context()) +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult + origin := keys.Origin(state.Context()) if origin.IsEmpty() { return r, errors.New("transaction has no origin [nonce]") } @@ -123,9 +110,9 @@ func (n *NonceHandler) Nonce( incrementNonceOnFailedTx := state.Config().GetNonceHandler().GetIncNonceOnFailedTx() if incrementNonceOnFailedTx && !isCheckTx { // Unconditionally increment the nonce in DeliverTx, regardless of whether the tx succeeds - seq = loomchain.NewSequence(nonceKey(origin)).Next(kvStore) + seq = sequence.NewSequence(nonceKey(origin)).Next(kvStore) } else { - seq = loomchain.NewSequence(nonceKey(origin)).Next(state) + seq = sequence.NewSequence(nonceKey(origin)).Next(state) } var tx NonceTx @@ -166,11 +153,11 @@ func (n *NonceHandler) Nonce( func (n *NonceHandler) IncNonce(state appstate.State, txBytes []byte, - result loomchain.TxHandlerResult, - postcommit loomchain.PostCommitHandler, + result txhandler.TxHandlerResult, + postcommit txhandler.PostCommitHandler, isCheckTx bool, ) error { - origin := Origin(state.Context()) + origin := keys.Origin(state.Context()) if origin.IsEmpty() { return errors.New("transaction has no origin [IncNonce]") } @@ -189,16 +176,16 @@ func (n *NonceHandler) IncNonce(state appstate.State, var NonceTxHandler = NonceHandler{nonceCache: make(map[string]uint64), lastHeight: 0} -var NonceTxPostNonceMiddleware = loomchain.PostCommitMiddlewareFunc(NonceTxHandler.IncNonce) +var NonceTxPostNonceMiddleware = txhandler.PostCommitMiddlewareFunc(NonceTxHandler.IncNonce) -var NonceTxMiddleware = func(kvStore store.KVStore) loomchain.TxMiddlewareFunc { +var NonceTxMiddleware = func(kvStore store.KVStore) txhandler.TxMiddlewareFunc { nonceTxMiddleware := func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (loomchain.TxHandlerResult, error) { + ) (txhandler.TxHandlerResult, error) { return NonceTxHandler.Nonce(state, kvStore, txBytes, next, isCheckTx) } - return loomchain.TxMiddlewareFunc(nonceTxMiddleware) + return txhandler.TxMiddlewareFunc(nonceTxMiddleware) } diff --git a/auth/auth_test.go b/auth/auth_test.go index 500bfb73ca..374e171b41 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -15,8 +15,10 @@ import ( "github.com/loomnetwork/go-loom/config" "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/auth/keys" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" ) func TestSignatureTxMiddleware(t *testing.T) { @@ -31,9 +33,9 @@ func TestSignatureTxMiddleware(t *testing.T) { require.NoError(t, err) state := appstate.NewStoreState(nil, store.NewMemStore(), abci.Header{}, nil, nil) SignatureTxMiddleware.ProcessTx(state, signedTxBytes, - func(state appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { + func(state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { require.Equal(t, txBytes, origBytes) - return loomchain.TxHandlerResult{}, nil + return txhandler.TxHandlerResult{}, nil }, false, ) } @@ -63,76 +65,76 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { cfg := config.DefaultConfig() cfg.NonceHandler.IncNonceOnFailedTx = true - ctx := context.WithValue(context.Background(), ContextKeyOrigin, origin) - ctx = context.WithValue(ctx, ContextKeyCheckTx, true) + ctx := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) + ctx = context.WithValue(ctx, keys.ContextKeyCheckTx, true) kvStore := store.NewMemStore() state := appstate.NewStoreState(ctx, kvStore, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, loomchain.TxHandlerResult{}, nil, false) + NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) //State is reset on every run - ctx2 := context.WithValue(context.Background(), ContextKeyOrigin, origin) + ctx2 := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) kvStore2 := store.NewMemStore() state2 := appstate.NewStoreState(ctx2, kvStore2, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) - _ = context.WithValue(ctx2, ContextKeyCheckTx, true) + _ = context.WithValue(ctx2, keys.ContextKeyCheckTx, true) //If we get the same sequence number in same block we should get an error _, err = NonceTxHandler.Nonce(state2, kvStore2, nonceTxBytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, true, ) require.Errorf(t, err, "sequence number does not match") // NonceTxPostNonceMiddleware shouldnt get called on an error //State is reset on every run - ctx3 := context.WithValue(context.Background(), ContextKeyOrigin, origin) + ctx3 := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) kvStore3 := store.NewMemStore() state3 := appstate.NewStoreState(ctx3, kvStore3, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) - _ = context.WithValue(ctx3, ContextKeyCheckTx, true) + _ = context.WithValue(ctx3, keys.ContextKeyCheckTx, true) //If we get to tx with incrementing sequence numbers we should be fine in the same block _, err = NonceTxHandler.Nonce(state3, kvStore3, nonceTxBytes2, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, true, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes2, loomchain.TxHandlerResult{}, nil, false) + NonceTxPostNonceMiddleware(state, nonceTxBytes2, txhandler.TxHandlerResult{}, nil, false) //Try a deliverTx at same height it should be fine - ctx3Dx := context.WithValue(context.Background(), ContextKeyOrigin, origin) + ctx3Dx := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) kvStore3Dx := store.NewMemStore() state3Dx := appstate.NewStoreState(ctx3Dx, kvStore3Dx, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) - _ = context.WithValue(ctx3Dx, ContextKeyCheckTx, true) + _ = context.WithValue(ctx3Dx, keys.ContextKeyCheckTx, true) _, err = NonceTxHandler.Nonce(state3Dx, kvStore3Dx, nonceTxBytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, loomchain.TxHandlerResult{}, nil, false) + NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) ///--------------increase block height should kill cache //State is reset on every run - ctx4 := context.WithValue(nil, ContextKeyOrigin, origin) + ctx4 := context.WithValue(nil, keys.ContextKeyOrigin, origin) kvStore4 := store.NewMemStore() state4 := appstate.NewStoreState(ctx4, kvStore4, abci.Header{Height: 28}, nil, nil).WithOnChainConfig(cfg) //If we get to tx with incrementing sequence numbers we should be fine in the same block _, err = NonceTxHandler.Nonce(state4, kvStore4, nonceTxBytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, true, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, loomchain.TxHandlerResult{}, nil, false) + NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) } func TestRevertedTxNonceMiddleware(t *testing.T) { @@ -161,8 +163,8 @@ func TestRevertedTxNonceMiddleware(t *testing.T) { Local: loom.LocalAddressFromPublicKey(pubkey), } - ctx := context.WithValue(context.Background(), ContextKeyOrigin, origin) - ctx = context.WithValue(ctx, ContextKeyCheckTx, true) + ctx := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) + ctx = context.WithValue(ctx, keys.ContextKeyCheckTx, true) kvStore := store.NewMemStore() storeTx := store.WrapAtomic(kvStore).BeginTx() state := appstate.NewStoreState(ctx, storeTx, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) @@ -173,19 +175,19 @@ func TestRevertedTxNonceMiddleware(t *testing.T) { // Send a successful tx _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, nil + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, loomchain.TxHandlerResult{}, nil, false) + NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) storeTx.Commit() storeTx.Rollback() // Send a failed tx, nonce should increase even though the transaction is reverted _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes2, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, errors.New("EVM transaction reverted") + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, errors.New("EVM transaction reverted") }, false, ) require.Error(t, err) @@ -205,8 +207,8 @@ func TestRevertedTxNonceMiddleware(t *testing.T) { // Send another failed tx, nonce should not increment because the transaction reverted _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes3, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { - return loomchain.TxHandlerResult{}, errors.New("EVM transaction reverted") + func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + return txhandler.TxHandlerResult{}, errors.New("EVM transaction reverted") }, false, ) require.Error(t, err) diff --git a/auth/chainconfig_middleware.go b/auth/chainconfig_middleware.go index c2eae4e54f..2a117a7c63 100644 --- a/auth/chainconfig_middleware.go +++ b/auth/chainconfig_middleware.go @@ -5,24 +5,25 @@ import ( loom "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" ) // NewChainConfigMiddleware returns middleware that verifies signed txs using either // SignedTxMiddleware or MultiChainSignatureTxMiddleware, it switches the underlying middleware // based on the on-chain and off-chain auth config settings. func NewChainConfigMiddleware( - authConfig *Config, + authConfig *keys.Config, createAddressMapperCtx func(state appstate.State) (contractpb.StaticContext, error), -) loomchain.TxMiddlewareFunc { - return loomchain.TxMiddlewareFunc(func( +) txhandler.TxMiddlewareFunc { + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (loomchain.TxHandlerResult, error) { + ) (txhandler.TxHandlerResult, error) { chains := getEnabledChains(authConfig.Chains, state) if len(chains) > 0 { mw := NewMultiChainSignatureTxMiddleware(chains, createAddressMapperCtx) @@ -34,8 +35,8 @@ func NewChainConfigMiddleware( } // Filters out any auth.ChainConfig(s) that haven't been enabled by the majority of validators. -func getEnabledChains(chains map[string]ChainConfig, state appstate.State) map[string]ChainConfig { - enabledChains := map[string]ChainConfig{} +func getEnabledChains(chains map[string]keys.ChainConfig, state appstate.State) map[string]keys.ChainConfig { + enabledChains := map[string]keys.ChainConfig{} for chainID, config := range chains { if state.FeatureEnabled(features.AuthSigTxFeaturePrefix+chainID, false) { enabledChains[chainID] = config @@ -46,9 +47,9 @@ func getEnabledChains(chains map[string]ChainConfig, state appstate.State) map[s if len(enabledChains) > 0 { curChainID := state.Block().ChainID if _, found := enabledChains[curChainID]; !found { - enabledChains[curChainID] = ChainConfig{ - TxType: LoomSignedTxType, - AccountType: NativeAccountType, + enabledChains[curChainID] = keys.ChainConfig{ + TxType: keys.LoomSignedTxType, + AccountType: keys.NativeAccountType, } } } @@ -58,7 +59,7 @@ func getEnabledChains(chains map[string]ChainConfig, state appstate.State) map[s // ResolveAccountAddress takes a local or foreign account address and returns the address used // to identify the account on this chain. func ResolveAccountAddress( - account loom.Address, state appstate.State, authCfg *Config, + account loom.Address, state appstate.State, authCfg *keys.Config, createAddressMapperCtx func(state appstate.State) (contractpb.StaticContext, error), ) (loom.Address, error) { chains := getEnabledChains(authCfg.Chains, state) @@ -69,10 +70,10 @@ func ResolveAccountAddress( } switch chain.AccountType { - case NativeAccountType: + case keys.NativeAccountType: return account, nil - case MappedAccountType: + case keys.MappedAccountType: addr, err := getMappedAccountAddress(state, account, createAddressMapperCtx) if err != nil { return loom.Address{}, err diff --git a/auth/chainconfig_middleware_test.go b/auth/chainconfig_middleware_test.go index 1450ca01d2..85d111435d 100644 --- a/auth/chainconfig_middleware_test.go +++ b/auth/chainconfig_middleware_test.go @@ -10,11 +10,14 @@ import ( "github.com/loomnetwork/go-loom/auth" goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/loomnetwork/loomchain" + + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/address_mapper" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" "golang.org/x/crypto/ed25519" @@ -34,15 +37,15 @@ func TestChainConfigMiddlewareSingleChain(t *testing.T) { fakeCtx := goloomplugin.CreateFakeContext(addr1, addr1) addresMapperAddr := fakeCtx.CreateContract(address_mapper.Contract) amCtx := contractpb.WrapPluginContext(fakeCtx.WithAddress(addresMapperAddr)) - authConfig := Config{ - Chains: map[string]ChainConfig{}, + authConfig := keys.Config{ + Chains: map[string]keys.ChainConfig{}, } chainConfigMiddleware := NewChainConfigMiddleware(&authConfig, func(state appstate.State) (contractpb.StaticContext, error) { return amCtx, nil }) _, err = chainConfigMiddleware.ProcessTx(state, signedTxBytes, - func(state appstate.State, txBytes []byte, isCheckTx bool) (loomchain.TxHandlerResult, error) { + func(state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { require.Equal(t, txBytes, origBytes) - return loomchain.TxHandlerResult{}, nil + return txhandler.TxHandlerResult{}, nil }, false, ) require.NoError(t, err) @@ -58,27 +61,27 @@ func TestChainConfigMiddlewareMultipleChain(t *testing.T) { addresMapperAddr := fakeCtx.CreateContract(address_mapper.Contract) amCtx := contractpb.WrapPluginContext(fakeCtx.WithAddress(addresMapperAddr)) - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) - chains := map[string]ChainConfig{ + chains := map[string]keys.ChainConfig{ "default": { - TxType: LoomSignedTxType, - AccountType: NativeAccountType, + TxType: keys.LoomSignedTxType, + AccountType: keys.NativeAccountType, }, "eth": { - TxType: EthereumSignedTxType, - AccountType: MappedAccountType, + TxType: keys.EthereumSignedTxType, + AccountType: keys.MappedAccountType, }, "tron": { - TxType: TronSignedTxType, - AccountType: MappedAccountType, + TxType: keys.TronSignedTxType, + AccountType: keys.MappedAccountType, }, "binance": { - TxType: BinanceSignedTxType, - AccountType: MappedAccountType, + TxType: keys.BinanceSignedTxType, + AccountType: keys.MappedAccountType, }, } - authConfig := Config{ + authConfig := keys.Config{ Chains: chains, } diff --git a/auth/config.go b/auth/keys/config.go similarity index 98% rename from auth/config.go rename to auth/keys/config.go index 582b4a74d6..ab9e53bd81 100644 --- a/auth/config.go +++ b/auth/keys/config.go @@ -1,4 +1,4 @@ -package auth +package keys type Config struct { // Per-chain tx signing config, indexed by chain ID diff --git a/auth/keys/keys.go b/auth/keys/keys.go new file mode 100644 index 0000000000..e92c964f54 --- /dev/null +++ b/auth/keys/keys.go @@ -0,0 +1,43 @@ +package keys + +import ( + "context" + + "github.com/loomnetwork/go-loom" +) + +type SignedTxType string + +const ( + LoomSignedTxType SignedTxType = "loom" + EthereumSignedTxType SignedTxType = "eth" + TronSignedTxType SignedTxType = "tron" + BinanceSignedTxType SignedTxType = "binance" +) + +// AccountType is used to specify which address should be used on-chain to identify a tx sender. +type AccountType int + +const ( + // NativeAccountType indicates that the tx sender address should be passed through to contracts + // as is, with the original chain ID intact. + NativeAccountType AccountType = iota + // MappedAccountType indicates that the tx sender address should be mapped to a DAppChain + // address before being passed through to contracts. + MappedAccountType +) + +type contextKey string + +func (c contextKey) String() string { + return "auth " + string(c) +} + +var ( + ContextKeyOrigin = contextKey("origin") + ContextKeyCheckTx = contextKey("CheckTx") +) + +func Origin(ctx context.Context) loom.Address { + return ctx.Value(ContextKeyOrigin).(loom.Address) +} diff --git a/auth/multi_chain_sigtx_middleware.go b/auth/multi_chain_sigtx_middleware.go index b14f56a122..3fc3189c78 100644 --- a/auth/multi_chain_sigtx_middleware.go +++ b/auth/multi_chain_sigtx_middleware.go @@ -7,45 +7,26 @@ import ( "fmt" "github.com/gogo/protobuf/proto" - loom "github.com/loomnetwork/go-loom" + "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/common/evmcompat" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" "github.com/loomnetwork/go-loom/vm" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/builtin/plugins/address_mapper" - "github.com/loomnetwork/loomchain/features" - appstate "github.com/loomnetwork/loomchain/state" "github.com/pkg/errors" "golang.org/x/crypto/ed25519" -) - -type SignedTxType string -const ( - LoomSignedTxType SignedTxType = "loom" - EthereumSignedTxType SignedTxType = "eth" - TronSignedTxType SignedTxType = "tron" - BinanceSignedTxType SignedTxType = "binance" -) - -// AccountType is used to specify which address should be used on-chain to identify a tx sender. -type AccountType int - -const ( - // NativeAccountType indicates that the tx sender address should be passed through to contracts - // as is, with the original chain ID intact. - NativeAccountType AccountType = iota - // MappedAccountType indicates that the tx sender address should be mapped to a DAppChain - // address before being passed through to contracts. - MappedAccountType + "github.com/loomnetwork/loomchain/auth/keys" + "github.com/loomnetwork/loomchain/builtin/plugins/address_mapper" + "github.com/loomnetwork/loomchain/features" + appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" ) -var originRecoveryFuncs = map[SignedTxType]originRecoveryFunc{ - LoomSignedTxType: verifyEd25519, - EthereumSignedTxType: verifySolidity66Byte, - TronSignedTxType: verifyTron, - BinanceSignedTxType: verifyBinance, +var originRecoveryFuncs = map[keys.SignedTxType]originRecoveryFunc{ + keys.LoomSignedTxType: verifyEd25519, + keys.EthereumSignedTxType: verifySolidity66Byte, + keys.TronSignedTxType: verifyTron, + keys.BinanceSignedTxType: verifyBinance, } type originRecoveryFunc func(tx SignedTx, allowedSigTypes []evmcompat.SignatureType) ([]byte, error) @@ -53,16 +34,16 @@ type originRecoveryFunc func(tx SignedTx, allowedSigTypes []evmcompat.SignatureT // NewMultiChainSignatureTxMiddleware returns tx signing middleware that supports a set of chain // specific signing algos. func NewMultiChainSignatureTxMiddleware( - chains map[string]ChainConfig, + chains map[string]keys.ChainConfig, createAddressMapperCtx func(state appstate.State) (contractpb.StaticContext, error), -) loomchain.TxMiddlewareFunc { - return loomchain.TxMiddlewareFunc(func( +) txhandler.TxMiddlewareFunc { + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult + ) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult var signedTx SignedTx if err := proto.Unmarshal(txBytes, &signedTx); err != nil { @@ -114,11 +95,11 @@ func NewMultiChainSignatureTxMiddleware( } switch chain.AccountType { - case NativeAccountType: // pass through origin & message sender as is - ctx := context.WithValue(state.Context(), ContextKeyOrigin, msgSender) + case keys.NativeAccountType: // pass through origin & message sender as is + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, msgSender) return next(state.WithContext(ctx), signedTx.Inner, isCheckTx) - case MappedAccountType: // map origin & message sender to an address on this chain + case keys.MappedAccountType: // map origin & message sender to an address on this chain origin, err := getMappedAccountAddress(state, msgSender, createAddressMapperCtx) if err != nil { return r, err @@ -142,7 +123,7 @@ func NewMultiChainSignatureTxMiddleware( return r, errors.Wrap(err, "failed to marshal NonceTx") } - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) return next(state.WithContext(ctx), nonceTxBytes, isCheckTx) default: diff --git a/auth/multi_chain_sigtx_middleware_test.go b/auth/multi_chain_sigtx_middleware_test.go index fa0e233d15..845324c19f 100644 --- a/auth/multi_chain_sigtx_middleware_test.go +++ b/auth/multi_chain_sigtx_middleware_test.go @@ -25,10 +25,12 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/address_mapper" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" ) const ( @@ -178,7 +180,7 @@ func TestEthAddressMappingVerification(t *testing.T) { addresMapperAddr := fakeCtx.CreateContract(address_mapper.Contract) amCtx := contractpb.WrapPluginContext(fakeCtx.WithAddress(addresMapperAddr)) - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) chains := map[string]ChainConfig{ "default": { @@ -247,7 +249,7 @@ func TestBinanceAddressMappingVerification(t *testing.T) { addresMapperAddr := fakeCtx.CreateContract(address_mapper.Contract) amCtx := contractpb.WrapPluginContext(fakeCtx.WithAddress(addresMapperAddr)) - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) chains := map[string]ChainConfig{ "default": { @@ -317,7 +319,7 @@ func TestChainIdVerification(t *testing.T) { addresMapperAddr := fakeCtx.CreateContract(address_mapper.Contract) amCtx := contractpb.WrapPluginContext(fakeCtx.WithAddress(addresMapperAddr)) - ctx := context.WithValue(state.Context(), ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) chains := map[string]ChainConfig{ "default": { @@ -379,16 +381,16 @@ func TestChainIdVerification(t *testing.T) { require.NoError(t, err) } -func throttleMiddlewareHandler(ttm loomchain.TxMiddlewareFunc, state appstate.State, signedTx []byte, ctx context.Context) (loomchain.TxHandlerResult, error) { +func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, signedTx []byte, ctx context.Context) (loomchain.TxHandlerResult, error) { return ttm.ProcessTx(state.WithContext(ctx), signedTx, - func(state appstate.State, txBytes []byte, isCheckTx bool) (res loomchain.TxHandlerResult, err error) { + func(state appstate.State, txBytes []byte, isCheckTx bool) (res txhandler.TxHandlerResult, err error) { var nonceTx NonceTx if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx loomchain.Transaction + var tx txhandler.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } @@ -397,12 +399,12 @@ func throttleMiddlewareHandler(ttm loomchain.TxMiddlewareFunc, state appstate.St return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) } - origin := Origin(state.Context()) + origin := keys.Origin(state.Context()) caller := loom.UnmarshalAddressPB(msg.From) if caller.Compare(origin) != 0 { return res, fmt.Errorf("Origin doesn't match caller: - %v != %v", origin, caller) } - return loomchain.TxHandlerResult{}, err + return txhandler.TxHandlerResult{}, err }, false, ) } @@ -461,7 +463,7 @@ func mockNonceTx(t *testing.T, from loom.Address, sequence uint64) []byte { From: from.MarshalPB(), }) require.NoError(t, err) - tx, err := proto.Marshal(&loomchain.Transaction{ + tx, err := proto.Marshal(&txhandler.Transaction{ Id: callId, Data: messageTx, }) diff --git a/sequence.go b/auth/sequence/sequence.go similarity index 97% rename from sequence.go rename to auth/sequence/sequence.go index 6986479ba2..5a084d3a63 100644 --- a/sequence.go +++ b/auth/sequence/sequence.go @@ -1,4 +1,4 @@ -package loomchain +package sequence import ( "bytes" diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index c4f425393d..c0bfa06bbd 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/prometheus/client_golang/prometheus/push" "github.com/tendermint/tendermint/libs/db" @@ -26,8 +27,9 @@ import ( "github.com/loomnetwork/go-loom/cli" "github.com/loomnetwork/go-loom/client" "github.com/loomnetwork/go-loom/crypto" - "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/util" + "github.com/prometheus/client_golang/prometheus" + "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/abci/backend" "github.com/loomnetwork/loomchain/auth" @@ -36,8 +38,17 @@ import ( plasmaConfig "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/config" plasmaOracle "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/oracle" "github.com/loomnetwork/loomchain/features" + "github.com/loomnetwork/loomchain/migrations" "github.com/loomnetwork/loomchain/receipts/leveldb" - "github.com/prometheus/client_golang/prometheus" + "github.com/loomnetwork/loomchain/throttle" + "github.com/loomnetwork/loomchain/tx_handler" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/txhandler/middleware" + + "github.com/pkg/errors" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cobra" + "golang.org/x/crypto/ed25519" "github.com/loomnetwork/loomchain/chainconfig" chaincfgcmd "github.com/loomnetwork/loomchain/cmd/loom/chainconfig" @@ -56,10 +67,9 @@ import ( "github.com/loomnetwork/loomchain/fnConsensus" karma_handler "github.com/loomnetwork/loomchain/karma" "github.com/loomnetwork/loomchain/log" - "github.com/loomnetwork/loomchain/migrations" "github.com/loomnetwork/loomchain/plugin" "github.com/loomnetwork/loomchain/receipts" - "github.com/loomnetwork/loomchain/receipts/handler" + rcommon "github.com/loomnetwork/loomchain/receipts/common" regcommon "github.com/loomnetwork/loomchain/registry" registry "github.com/loomnetwork/loomchain/registry/factory" "github.com/loomnetwork/loomchain/rpc" @@ -67,13 +77,7 @@ import ( "github.com/loomnetwork/loomchain/store" blockindex "github.com/loomnetwork/loomchain/store/block_index" evmaux "github.com/loomnetwork/loomchain/store/evm_aux" - "github.com/loomnetwork/loomchain/throttle" - "github.com/loomnetwork/loomchain/tx_handler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" - stdprometheus "github.com/prometheus/client_golang/prometheus" - "github.com/spf13/cobra" - "golang.org/x/crypto/ed25519" ) var ( @@ -589,7 +593,7 @@ func destroyApp(cfg *config.Config) error { } func destroyReceiptsDB(cfg *config.Config) { - if cfg.ReceiptsVersion == handler.ReceiptHandlerLevelDb || cfg.ReceiptsVersion == 3 { + if cfg.ReceiptsVersion == rcommon.ReceiptHandlerLevelDb || cfg.ReceiptsVersion == 3 { receptHandler := leveldb.LevelDbReceipts{} receptHandler.ClearData() } @@ -825,26 +829,6 @@ func loadApp( } evm.LogEthDbBatch = cfg.LogEthDbBatch - deployTxHandler := &vm.DeployTxHandler{ - Manager: vmManager, - CreateRegistry: createRegistry, - AllowNamedEVMContracts: cfg.AllowNamedEvmContracts, - } - - callTxHandler := &vm.CallTxHandler{ - Manager: vmManager, - } - - migrationTxHandler := &tx_handler.MigrationTxHandler{ - Manager: vmManager, - CreateRegistry: createRegistry, - Migrations: map[int32]tx_handler.MigrationFunc{ - 1: migrations.DPOSv3Migration, - 2: migrations.GatewayMigration, - 3: migrations.GatewayMigration, - }, - } - gen, err := config.ReadGenesis(cfg.GenesisPath()) if err != nil { return nil, err @@ -878,105 +862,8 @@ func loadApp( return nil } - router := loomchain.NewTxRouter() - - isEvmTx := func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) bool { - var msg vm.MessageTx - err := proto.Unmarshal(txBytes, &msg) - if err != nil { - return false - } - - switch txID { - case 1: - var tx vm.DeployTx - err = proto.Unmarshal(msg.Data, &tx) - if err != nil { - // In case of error, let's give safest response, - // let's TxHandler down the line, handle it. - return false - } - return tx.VmType == vm.VMType_EVM - case 2: - var tx vm.CallTx - err = proto.Unmarshal(msg.Data, &tx) - if err != nil { - // In case of error, let's give safest response, - // let's TxHandler down the line, handle it. - return false - } - return tx.VmType == vm.VMType_EVM - case 3: - return false - default: - return false - } - } - - router.HandleDeliverTx(1, loomchain.GeneratePassthroughRouteHandler(deployTxHandler)) - router.HandleDeliverTx(2, loomchain.GeneratePassthroughRouteHandler(callTxHandler)) - router.HandleDeliverTx(3, loomchain.GeneratePassthroughRouteHandler(migrationTxHandler)) - - // TODO: Write this in more elegant way - router.HandleCheckTx(1, loomchain.GenerateConditionalRouteHandler(isEvmTx, loomchain.NoopTxHandler, deployTxHandler)) - router.HandleCheckTx(2, loomchain.GenerateConditionalRouteHandler(isEvmTx, loomchain.NoopTxHandler, callTxHandler)) - router.HandleCheckTx(3, loomchain.GenerateConditionalRouteHandler(isEvmTx, loomchain.NoopTxHandler, migrationTxHandler)) - - txMiddleWare := []loomchain.TxMiddleware{ - loomchain.LogTxMiddleware, - loomchain.RecoveryTxMiddleware, - } - - postCommitMiddlewares := []loomchain.PostCommitMiddleware{ - loomchain.LogPostCommitMiddleware, - } - - txMiddleWare = append(txMiddleWare, auth.NewChainConfigMiddleware( - cfg.Auth, - getContractStaticCtx("addressmapper", vmManager), - )) - createKarmaContractCtx := getContractCtx("karma", vmManager) - if cfg.Karma.Enabled { - txMiddleWare = append(txMiddleWare, throttle.GetKarmaMiddleWare( - cfg.Karma.Enabled, - cfg.Karma.MaxCallCount, - cfg.Karma.SessionDuration, - createKarmaContractCtx, - )) - } - - if cfg.TxLimiter.Enabled { - txMiddleWare = append(txMiddleWare, throttle.NewTxLimiterMiddleware(cfg.TxLimiter)) - } - - if cfg.ContractTxLimiter.Enabled { - contextFactory := getContractCtx("user-deployer-whitelist", vmManager) - txMiddleWare = append( - txMiddleWare, throttle.NewContractTxLimiterMiddleware(cfg.ContractTxLimiter, contextFactory), - ) - } - - if cfg.DeployerWhitelist.ContractEnabled { - contextFactory := getContractCtx("deployerwhitelist", vmManager) - dwMiddleware, err := throttle.NewDeployerWhitelistMiddleware(contextFactory) - if err != nil { - return nil, err - } - txMiddleWare = append(txMiddleWare, dwMiddleware) - - } - - if cfg.UserDeployerWhitelist.ContractEnabled { - contextFactory := getContractCtx("user-deployer-whitelist", vmManager) - evmDeployRecorderMiddleware, err := throttle.NewEVMDeployRecorderPostCommitMiddleware(contextFactory) - if err != nil { - return nil, err - } - postCommitMiddlewares = append(postCommitMiddlewares, evmDeployRecorderMiddleware) - } - createContractUpkeepHandler := func(state appstate.State) (loomchain.KarmaHandler, error) { // TODO: This setting should be part of the config stored within the Karma contract itself, // that will allow us to switch the upkeep on & off via a tx. @@ -1024,18 +911,6 @@ func loadApp( return loom.NewValidatorSet(b.GenesisValidators()...), nil } - txMiddleWare = append(txMiddleWare, auth.NonceTxMiddleware(appStore)) - - if cfg.GoContractDeployerWhitelist.Enabled { - goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) - if err != nil { - return nil, errors.Wrapf(err, "getting list of users allowed go deploys") - } - txMiddleWare = append(txMiddleWare, throttle.GetGoDeployTxMiddleWare(goDeployers)) - } - - txMiddleWare = append(txMiddleWare, loomchain.NewInstrumentingTxMiddleware()) - createValidatorsManager := func(state appstate.State) (loomchain.ValidatorsManager, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) if err != nil { @@ -1089,17 +964,33 @@ func loadApp( return nil, err } } + /* + txMiddleWare, err := middleware.TxMiddleWare(cfg, vmManager, chainID, appStore) + if err != nil { + return nil, err + } + */ + postCommitMiddlewares, err := postCommitMiddleWAre(cfg, vmManager) + if err != nil { + return nil, err + } - // We need to make sure nonce post commit middleware is last - // as it doesn't pass control to other middlewares after it. - postCommitMiddlewares = append(postCommitMiddlewares, auth.NonceTxPostNonceMiddleware) + //txHandler, err := middleware.AppTxHandler(cfg, vmManager, createRegistry, chainID, appStore) + //if err != nil { + // return nil, err + //} + + txMiddleware, err := txMiddleWare(cfg, vmManager, chainID, appStore) + if err != nil { + return nil, err + } return &loomchain.Application{ Store: appStore, Init: init, - TxHandler: loomchain.MiddlewareTxHandler( - txMiddleWare, - router, + TxHandler: txhandler.MiddlewareTxHandler( + txMiddleware, + router(cfg, vmManager, createRegistry), postCommitMiddlewares, ), BlockIndexStore: blockIndexStore, @@ -1159,11 +1050,163 @@ func deployContract( return nil } +func txMiddleWare( + cfg *config.Config, + vmManager *vm.Manager, + chainID string, + appStore store.VersionedKVStore, +) ([]txhandler.TxMiddleware, error) { + txMiddleWare := []txhandler.TxMiddleware{ + txhandler.LogTxMiddleware, + txhandler.RecoveryTxMiddleware, + } + + txMiddleWare = append(txMiddleWare, auth.NewChainConfigMiddleware( + cfg.Auth, + getContractStaticCtx("addressmapper", vmManager), + )) + + if cfg.Karma.Enabled { + txMiddleWare = append(txMiddleWare, throttle.GetKarmaMiddleWare( + cfg.Karma.Enabled, + cfg.Karma.MaxCallCount, + cfg.Karma.SessionDuration, + getContractCtx("karma", vmManager), + )) + } + + if cfg.TxLimiter.Enabled { + txMiddleWare = append(txMiddleWare, middleware.NewTxLimiterMiddleware(cfg.TxLimiter)) + } + + if cfg.ContractTxLimiter.Enabled { + udwCtxFactory := getContractCtx("user-deployer-whitelist", vmManager) + txMiddleWare = append( + txMiddleWare, middleware.NewContractTxLimiterMiddleware(cfg.ContractTxLimiter, udwCtxFactory), + ) + } + + if cfg.DeployerWhitelist.ContractEnabled { + dwMiddleware, err := middleware.NewDeployerWhitelistMiddleware(getContractCtx("deployerwhitelist", vmManager)) + if err != nil { + return nil, err + } + txMiddleWare = append(txMiddleWare, dwMiddleware) + + } + + txMiddleWare = append(txMiddleWare, auth.NonceTxMiddleware(appStore)) + + if cfg.GoContractDeployerWhitelist.Enabled { + goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) + if err != nil { + return nil, errors.Wrapf(err, "getting list of users allowed go deploys") + } + txMiddleWare = append(txMiddleWare, throttle.GetGoDeployTxMiddleWare(goDeployers)) + } + + txMiddleWare = append(txMiddleWare, txhandler.NewInstrumentingTxMiddleware()) + + return txMiddleWare, nil +} + +func router( + cfg *config.Config, + vmManager *vm.Manager, + createRegistry registry.RegistryFactoryFunc, +) txhandler.TxHandler { + router := middleware.NewTxRouter() + isEvmTx := func(txID uint32, _ appstate.State, txBytes []byte, isCheckTx bool) bool { + var msg vm.MessageTx + err := proto.Unmarshal(txBytes, &msg) + if err != nil { + return false + } + + switch txID { + case 1: + var tx vm.DeployTx + err = proto.Unmarshal(msg.Data, &tx) + if err != nil { + // In case of error, let's give safest response, + // let's TxHandler down the line, handle it. + return false + } + return tx.VmType == vm.VMType_EVM + case 2: + var tx vm.CallTx + err = proto.Unmarshal(msg.Data, &tx) + if err != nil { + // In case of error, let's give safest response, + // let's TxHandler down the line, handle it. + return false + } + return tx.VmType == vm.VMType_EVM + case 3: + return false + default: + return false + } + } + + deployTxHandler := &vm.DeployTxHandler{ + Manager: vmManager, + CreateRegistry: createRegistry, + AllowNamedEVMContracts: cfg.AllowNamedEvmContracts, + } + + callTxHandler := &vm.CallTxHandler{ + Manager: vmManager, + } + + migrationTxHandler := &tx_handler.MigrationTxHandler{ + Manager: vmManager, + CreateRegistry: createRegistry, + Migrations: map[int32]tx_handler.MigrationFunc{ + 1: migrations.DPOSv3Migration, + 2: migrations.GatewayMigration, + 3: migrations.GatewayMigration, + }, + } + + router.HandleDeliverTx(1, middleware.GeneratePassthroughRouteHandler(deployTxHandler)) + router.HandleDeliverTx(2, middleware.GeneratePassthroughRouteHandler(callTxHandler)) + router.HandleDeliverTx(3, middleware.GeneratePassthroughRouteHandler(migrationTxHandler)) + + // TODO: Write this in more elegant way + router.HandleCheckTx(1, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, deployTxHandler)) + router.HandleCheckTx(2, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, callTxHandler)) + router.HandleCheckTx(3, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, migrationTxHandler)) + + return router +} + +func postCommitMiddleWAre(cfg *config.Config, vmManager *vm.Manager) ([]txhandler.PostCommitMiddleware, error) { + postCommitMiddlewares := []txhandler.PostCommitMiddleware{ + txhandler.LogPostCommitMiddleware, + } + + if cfg.UserDeployerWhitelist.ContractEnabled { + udwContractFactory := getContractCtx("user-deployer-whitelist", vmManager) + evmDeployRecorderMiddleware, err := middleware.NewEVMDeployRecorderPostCommitMiddleware(udwContractFactory) + if err != nil { + return nil, err + } + postCommitMiddlewares = append(postCommitMiddlewares, evmDeployRecorderMiddleware) + } + + // We need to make sure nonce post commit middleware is last as + // it doesn't pass control to other middlewares after it. + postCommitMiddlewares = append(postCommitMiddlewares, auth.NonceTxPostNonceMiddleware) + + return postCommitMiddlewares, nil +} + type contextFactory func(state appstate.State) (contractpb.Context, error) type staticContextFactory func(state appstate.State) (contractpb.StaticContext, error) -func getContractCtx(pluginName string, vmManager *vm.Manager) contextFactory { +func getContractCtx(pluginName string, vmManager *vm.Manager) func(state appstate.State) (contractpb.Context, error) { return func(state appstate.State) (contractpb.Context, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) if err != nil { @@ -1173,7 +1216,7 @@ func getContractCtx(pluginName string, vmManager *vm.Manager) contextFactory { } } -func getContractStaticCtx(pluginName string, vmManager *vm.Manager) staticContextFactory { +func getContractStaticCtx(pluginName string, vmManager *vm.Manager) func(state appstate.State) (contractpb.StaticContext, error) { return func(state appstate.State) (contractpb.StaticContext, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) if err != nil { diff --git a/config/config.go b/config/config.go index da1040aeb3..a26cf6947c 100755 --- a/config/config.go +++ b/config/config.go @@ -10,19 +10,18 @@ import ( "path" "path/filepath" - "github.com/loomnetwork/loomchain/auth" + "github.com/pkg/errors" + "github.com/spf13/viper" + + "github.com/loomnetwork/loomchain/auth/keys" plasmacfg "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/config" genesiscfg "github.com/loomnetwork/loomchain/config/genesis" "github.com/loomnetwork/loomchain/events" - "github.com/loomnetwork/loomchain/evm" hsmpv "github.com/loomnetwork/loomchain/privval/hsm" - receipts "github.com/loomnetwork/loomchain/receipts/handler" + "github.com/loomnetwork/loomchain/receipts/common" registry "github.com/loomnetwork/loomchain/registry/factory" "github.com/loomnetwork/loomchain/store" blockindex "github.com/loomnetwork/loomchain/store/block_index" - "github.com/loomnetwork/loomchain/throttle" - "github.com/pkg/errors" - "github.com/spf13/viper" "github.com/loomnetwork/loomchain/db" @@ -67,9 +66,9 @@ type Config struct { CallEnabled bool SessionDuration int64 Karma *KarmaConfig - GoContractDeployerWhitelist *throttle.GoContractDeployerWhitelistConfig - TxLimiter *throttle.TxLimiterConfig - ContractTxLimiter *throttle.ContractTxLimiterConfig + GoContractDeployerWhitelist *GoContractDeployerWhitelistConfig + TxLimiter *TxLimiterConfig + ContractTxLimiter *ContractTxLimiterConfig // Logging LogDestination string ContractLogLevel string @@ -133,9 +132,9 @@ type Config struct { FnConsensus *FnConsensusConfig - Auth *auth.Config + Auth *keys.Config - EvmStore *evm.EvmStoreConfig + EvmStore *EvmStoreConfig // Allow deployment of named EVM contracts (should only be used in tests!) AllowNamedEvmContracts bool @@ -376,8 +375,8 @@ func DefaultConfig() *Config { LogStateDB: false, LogEthDbBatch: false, RegistryVersion: int32(registry.RegistryV2), - ReceiptsVersion: int32(receipts.ReceiptHandlerLevelDb), - EVMPersistentTxReceiptsMax: receipts.DefaultMaxReceipts, + ReceiptsVersion: int32(common.ReceiptHandlerLevelDb), + EVMPersistentTxReceiptsMax: common.DefaultMaxReceipts, SessionDuration: 600, EVMAccountsEnabled: false, EVMDebugEnabled: false, @@ -396,9 +395,9 @@ func DefaultConfig() *Config { cfg.PlasmaCash = plasmacfg.DefaultConfig() cfg.AppStore = store.DefaultConfig() cfg.HsmConfig = hsmpv.DefaultConfig() - cfg.TxLimiter = throttle.DefaultTxLimiterConfig() - cfg.ContractTxLimiter = throttle.DefaultContractTxLimiterConfig() - cfg.GoContractDeployerWhitelist = throttle.DefaultGoContractDeployerWhitelistConfig() + cfg.TxLimiter = DefaultTxLimiterConfig() + cfg.ContractTxLimiter = DefaultContractTxLimiterConfig() + cfg.GoContractDeployerWhitelist = DefaultGoContractDeployerWhitelistConfig() cfg.DPOSv2OracleConfig = DefaultDPOS2OracleConfig() cfg.CachingStoreConfig = store.DefaultCachingStoreConfig() cfg.BlockStore = store.DefaultBlockStoreConfig() @@ -412,11 +411,11 @@ func DefaultConfig() *Config { cfg.PrometheusPushGateway = DefaultPrometheusPushGatewayConfig() cfg.EventDispatcher = events.DefaultEventDispatcherConfig() cfg.EventStore = events.DefaultEventStoreConfig() - cfg.EvmStore = evm.DefaultEvmStoreConfig() + cfg.EvmStore = DefaultEvmStoreConfig() cfg.FnConsensus = DefaultFnConsensusConfig() - cfg.Auth = auth.DefaultConfig() + cfg.Auth = keys.DefaultConfig() return cfg } diff --git a/evm/config.go b/config/evm_config.go similarity index 98% rename from evm/config.go rename to config/evm_config.go index 4239a74b11..47977012c1 100644 --- a/evm/config.go +++ b/config/evm_config.go @@ -1,4 +1,4 @@ -package evm +package config type EvmStoreConfig struct { // DBName defines database file name diff --git a/config/throttle_config.go b/config/throttle_config.go new file mode 100644 index 0000000000..fe5664b399 --- /dev/null +++ b/config/throttle_config.go @@ -0,0 +1,82 @@ +package config + +import ( + "github.com/loomnetwork/go-loom" + "github.com/pkg/errors" +) + +type GoContractDeployerWhitelistConfig struct { + Enabled bool + DeployerAddressList []string +} + +func DefaultGoContractDeployerWhitelistConfig() *GoContractDeployerWhitelistConfig { + return &GoContractDeployerWhitelistConfig{ + Enabled: false, + } +} + +func (c *GoContractDeployerWhitelistConfig) DeployerAddresses(chainId string) ([]loom.Address, error) { + deployerAddressList := make([]loom.Address, 0, len(c.DeployerAddressList)) + for _, addrStr := range c.DeployerAddressList { + addr, err := loom.ParseAddress(addrStr) + if err != nil { + addr, err = loom.ParseAddress(chainId + ":" + addrStr) + if err != nil { + return nil, errors.Wrapf(err, "parsing deploy address %s", addrStr) + } + } + deployerAddressList = append(deployerAddressList, addr) + } + return deployerAddressList, nil +} + +type TxLimiterConfig struct { + // Enables the tx limiter middleware + Enabled bool + // Number of seconds each session lasts + SessionDuration int64 + // Maximum number of txs that should be allowed per session + MaxTxsPerSession int64 +} + +func DefaultTxLimiterConfig() *TxLimiterConfig { + return &TxLimiterConfig{ + SessionDuration: 60, + MaxTxsPerSession: 60, + } +} + +// Clone returns a deep clone of the config. +func (c *TxLimiterConfig) Clone() *TxLimiterConfig { + if c == nil { + return nil + } + clone := *c + return &clone +} + +type ContractTxLimiterConfig struct { + // Enables the middleware + Enabled bool + // Number of seconds each refresh lasts + ContractDataRefreshInterval int64 + TierDataRefreshInterval int64 +} + +func DefaultContractTxLimiterConfig() *ContractTxLimiterConfig { + return &ContractTxLimiterConfig{ + Enabled: false, + ContractDataRefreshInterval: 15 * 60, + TierDataRefreshInterval: 15 * 60, + } +} + +// Clone returns a deep clone of the config. +func (c *ContractTxLimiterConfig) Clone() *ContractTxLimiterConfig { + if c == nil { + return nil + } + clone := *c + return &clone +} diff --git a/eth/polls/polls_test.go b/eth/polls/polls_test.go index 60fff884b9..07c5b10e4f 100644 --- a/eth/polls/polls_test.go +++ b/eth/polls/polls_test.go @@ -28,7 +28,7 @@ var ( ) func TestLogPoll(t *testing.T) { - testLogPoll(t, handler.ReceiptHandlerLevelDb) + testLogPoll(t, common.ReceiptHandlerLevelDb) } func testLogPoll(t *testing.T, version handler.ReceiptHandlerVersion) { @@ -37,7 +37,7 @@ func testLogPoll(t *testing.T, version handler.ReceiptHandlerVersion) { blockStore := store.NewMockBlockStore() eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) sub := NewEthSubscriptions(evmAuxStore, blockStore) allFilter := eth.JsonFilter{ FromBlock: "earliest", @@ -88,8 +88,8 @@ func testLogPoll(t *testing.T, version handler.ReceiptHandlerVersion) { } func TestTxPoll(t *testing.T) { - testLegacyTxPoll(t, handler.ReceiptHandlerLevelDb) - testTxPoll(t, handler.ReceiptHandlerLevelDb) + testLegacyTxPoll(t, common.ReceiptHandlerLevelDb) + testTxPoll(t, common.ReceiptHandlerLevelDb) } func testLegacyTxPoll(t *testing.T, version handler.ReceiptHandlerVersion) { @@ -98,7 +98,7 @@ func testLegacyTxPoll(t *testing.T, version handler.ReceiptHandlerVersion) { blockStore := store.NewMockBlockStore() eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) sub := NewEthSubscriptions(evmAuxStore, blockStore) state := makeMockState(t, receiptHandler) @@ -137,7 +137,7 @@ func testTxPoll(t *testing.T, version handler.ReceiptHandlerVersion) { blockStore := store.NewMockBlockStore() eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) sub := NewEthSubscriptions(evmAuxStore, blockStore) state := makeMockState(t, receiptHandler) @@ -203,7 +203,7 @@ func testTxPoll(t *testing.T, version handler.ReceiptHandlerVersion) { } func TestTimeout(t *testing.T) { - testTimeout(t, handler.ReceiptHandlerLevelDb) + testTimeout(t, common.ReceiptHandlerLevelDb) } func testTimeout(t *testing.T, version handler.ReceiptHandlerVersion) { @@ -212,7 +212,7 @@ func testTimeout(t *testing.T, version handler.ReceiptHandlerVersion) { blockStore := store.NewMockBlockStore() eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) BlockTimeout = 10 sub := NewEthSubscriptions(evmAuxStore, blockStore) diff --git a/eth/query/block.go b/eth/query/block.go index 6edaaf990e..c9344ec08d 100644 --- a/eth/query/block.go +++ b/eth/query/block.go @@ -12,6 +12,7 @@ import ( "github.com/loomnetwork/go-loom/auth" "github.com/loomnetwork/go-loom/plugin/types" + ltypes "github.com/loomnetwork/go-loom/types" "github.com/loomnetwork/go-loom/vm" "github.com/loomnetwork/loomchain" @@ -145,7 +146,7 @@ func GetTxObjectFromBlockResult( } txObj.Nonce = eth.EncInt(int64(nonceTx.Sequence)) - var txTx loomchain.Transaction + var txTx ltypes.Transaction if err := proto.Unmarshal(nonceTx.Inner, &txTx); err != nil { return eth.GetEmptyTxObject(), nil, err } diff --git a/eth/query/query_test.go b/eth/query/query_test.go index b2fe6aabef..aab49d597e 100644 --- a/eth/query/query_test.go +++ b/eth/query/query_test.go @@ -33,7 +33,7 @@ var ( ) func TestQueryChain(t *testing.T) { - testQueryChain(t, handler.ReceiptHandlerLevelDb) + testQueryChain(t, common.ReceiptHandlerLevelDb) } func testQueryChain(t *testing.T, v handler.ReceiptHandlerVersion) { @@ -41,7 +41,7 @@ func testQueryChain(t *testing.T, v handler.ReceiptHandlerVersion) { require.NoError(t, err) eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) var writer loomchain.WriteReceiptHandler = receiptHandler require.NoError(t, err) @@ -145,7 +145,7 @@ func TestMatchFilters(t *testing.T) { ethFilter5 := eth.EthBlockFilter{ Topics: [][]string{{"Topic1"}, {"Topic6"}}, } - bloomFilter := bloom.GenBloomFilter(common.ConvertEventData(testEvents)) + bloomFilter := bloom.GenBloomFilter(ConvertEventData(testEvents)) require.True(t, MatchBloomFilter(ethFilter1, bloomFilter)) require.False(t, MatchBloomFilter(ethFilter2, bloomFilter)) // address does not match @@ -167,7 +167,7 @@ func TestMatchFilters(t *testing.T) { } func TestGetLogs(t *testing.T) { - testGetLogs(t, handler.ReceiptHandlerLevelDb) + testGetLogs(t, common.ReceiptHandlerLevelDb) } func testGetLogs(t *testing.T, v handler.ReceiptHandlerVersion) { @@ -176,7 +176,7 @@ func testGetLogs(t *testing.T, v handler.ReceiptHandlerVersion) { eventDispatcher := events.NewLogEventDispatcher() eventHandler := loomchain.NewDefaultEventHandler(eventDispatcher) - receiptHandler := handler.NewReceiptHandler(eventHandler, handler.DefaultMaxReceipts, evmAuxStore) + receiptHandler := handler.NewReceiptHandler(eventHandler, common.DefaultMaxReceipts, evmAuxStore) var writer loomchain.WriteReceiptHandler = receiptHandler require.NoError(t, err) @@ -233,3 +233,12 @@ func testGetLogs(t *testing.T, v handler.ReceiptHandlerVersion) { require.NoError(t, receiptHandler.Close()) } + +func ConvertEventData(events []*loomchain.EventData) []*types.EventData { + typesEvents := make([]*types.EventData, 0, len(events)) + for _, event := range events { + typeEvent := types.EventData(*event) + typesEvents = append(typesEvents, &typeEvent) + } + return typesEvents +} diff --git a/events/db_indexer.go b/events/db_indexer.go index 945376ca99..5c8561a4a0 100644 --- a/events/db_indexer.go +++ b/events/db_indexer.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/loomnetwork/go-loom/plugin/types" - "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/store" ) @@ -16,7 +16,7 @@ type DBIndexerEventDispatcher struct { sync.Mutex } -var _ loomchain.EventDispatcher = &DBIndexerEventDispatcher{} +//var _ loomchain.EventDispatcher = &DBIndexerEventDispatcher{} func NewDBIndexerEventDispatcher(es store.EventStore) *DBIndexerEventDispatcher { return &DBIndexerEventDispatcher{EventStore: es} diff --git a/events/log.go b/events/log.go index 1959176aab..b93e218845 100644 --- a/events/log.go +++ b/events/log.go @@ -1,15 +1,14 @@ package events import ( - "github.com/loomnetwork/loomchain" - log "github.com/loomnetwork/loomchain/log" + "github.com/loomnetwork/loomchain/log" ) // LogEventDispatcher just logs events type LogEventDispatcher struct { } -var _ loomchain.EventDispatcher = &LogEventDispatcher{} +//var _ loomchain.EventDispatcher = &LogEventDispatcher{} // NewLogEventDispatcher create a new redis dispatcher func NewLogEventDispatcher() *LogEventDispatcher { diff --git a/events/redis.go b/events/redis.go index 66400e5dad..d253d97b0f 100644 --- a/events/redis.go +++ b/events/redis.go @@ -1,8 +1,7 @@ package events import ( - "github.com/loomnetwork/loomchain" - log "github.com/loomnetwork/loomchain/log" + "github.com/loomnetwork/loomchain/log" "github.com/gomodule/redigo/redis" ) @@ -17,7 +16,7 @@ type RedisEventDispatcher struct { queue string } -var _ loomchain.EventDispatcher = &RedisEventDispatcher{} +//var _ loomchain.EventDispatcher = &RedisEventDispatcher{} // NewRedisEventDispatcher create a new redis dispatcher func NewRedisEventDispatcher(host string) (*RedisEventDispatcher, error) { diff --git a/evm/loomevm.go b/evm/loomevm.go index fa7edd24f7..555f424f93 100644 --- a/evm/loomevm.go +++ b/evm/loomevm.go @@ -22,7 +22,7 @@ import ( "github.com/loomnetwork/loomchain/events" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/receipts" - "github.com/loomnetwork/loomchain/receipts/handler" + rcommon "github.com/loomnetwork/loomchain/receipts/common" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/vm" ) @@ -108,7 +108,7 @@ var LoomVmFactory = func(state appstate.State) (vm.VM, error) { eventHandler := loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher()) receiptHandlerProvider := receipts.NewReceiptHandlerProvider( eventHandler, - handler.DefaultMaxReceipts, + rcommon.DefaultMaxReceipts, nil, ) receiptHandler := receiptHandlerProvider.Writer() diff --git a/fork.go b/fork.go index 43ac29489d..cbaaebb70d 100644 --- a/fork.go +++ b/fork.go @@ -4,10 +4,11 @@ import ( "sort" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" ) type forkRoute struct { - TxHandler + txhandler.TxHandler Height int64 } @@ -35,7 +36,7 @@ func NewForkRouter() *ForkRouter { } } -func (r *ForkRouter) Handle(chainID string, height int64, handler TxHandler) { +func (r *ForkRouter) Handle(chainID string, height int64, handler txhandler.TxHandler) { routes := r.routes[chainID] found := sort.Search(len(routes), func(i int) bool { return routes[i].Height >= height @@ -52,11 +53,11 @@ func (r *ForkRouter) Handle(chainID string, height int64, handler TxHandler) { r.routes[chainID] = routes } -func (r *ForkRouter) ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { +func (r *ForkRouter) ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { block := state.Block() routes := r.routes[block.ChainID] - var found TxHandler + var found txhandler.TxHandler for _, route := range routes { if route.Height > block.Height { break diff --git a/plugin/vm_test.go b/plugin/vm_test.go index 4911bfb29c..3875f7124c 100644 --- a/plugin/vm_test.go +++ b/plugin/vm_test.go @@ -210,7 +210,7 @@ func TestGetEvmTxReceipt(t *testing.T) { require.NoError(t, err) receiptHandler := handler.NewReceiptHandler( loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher()), - handler.DefaultMaxReceipts, + common.DefaultMaxReceipts, evmAuxStore, ) require.NoError(t, err) @@ -240,7 +240,7 @@ func TestGetEvmTxReceiptNoCommit(t *testing.T) { require.NoError(t, err) receiptHandler := handler.NewReceiptHandler( loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher()), - handler.DefaultMaxReceipts, + common.DefaultMaxReceipts, evmAuxStore, ) diff --git a/receipts/common/common.go b/receipts/common/common.go index 7383c4c94e..48ce128218 100644 --- a/receipts/common/common.go +++ b/receipts/common/common.go @@ -3,14 +3,14 @@ package common import ( "encoding/binary" - "github.com/loomnetwork/go-loom/plugin/types" - "github.com/loomnetwork/loomchain" "github.com/pkg/errors" ) const ( - StatusTxSuccess = int32(1) - StatusTxFail = int32(0) + StatusTxSuccess = int32(1) + StatusTxFail = int32(0) + ReceiptHandlerLevelDb = 2 //ctypes.ReceiptStorage_LEVELDB + DefaultMaxReceipts = uint64(2000) ) var ( @@ -23,12 +23,3 @@ func BlockHeightToBytes(height uint64) []byte { binary.LittleEndian.PutUint64(heightB, height) return heightB } - -func ConvertEventData(events []*loomchain.EventData) []*types.EventData { - typesEvents := make([]*types.EventData, 0, len(events)) - for _, event := range events { - typeEvent := types.EventData(*event) - typesEvents = append(typesEvents, &typeEvent) - } - return typesEvents -} diff --git a/receipts/handler/handler.go b/receipts/handler/handler.go index 16640cf97a..e6a2f07053 100644 --- a/receipts/handler/handler.go +++ b/receipts/handler/handler.go @@ -18,11 +18,6 @@ import ( type ReceiptHandlerVersion int32 -const ( - ReceiptHandlerLevelDb = 2 //ctypes.ReceiptStorage_LEVELDB - DefaultMaxReceipts = uint64(2000) -) - // ReceiptHandler implements loomchain.ReadReceiptHandler, loomchain.WriteReceiptHandler, and // loomchain.ReceiptHandlerStore interfaces. type ReceiptHandler struct { diff --git a/receipts/handler/handler_test.go b/receipts/handler/handler_test.go index fc0b1d3f27..4e8dbe5892 100644 --- a/receipts/handler/handler_test.go +++ b/receipts/handler/handler_test.go @@ -9,6 +9,7 @@ import ( "github.com/loomnetwork/go-loom/plugin/types" "github.com/loomnetwork/go-loom/util" "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/auth/sequence" "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/receipts/common" "github.com/stretchr/testify/require" @@ -27,7 +28,7 @@ func TestReceiptsHandlerChain(t *testing.T) { evmAuxStore, err := common.NewMockEvmAuxStore() require.NoError(t, err) - handler := NewReceiptHandler(&loomchain.DefaultEventHandler{}, DefaultMaxReceipts, evmAuxStore) + handler := NewReceiptHandler(&loomchain.DefaultEventHandler{}, common.DefaultMaxReceipts, evmAuxStore) var writer loomchain.WriteReceiptHandler = handler var reader loomchain.ReadReceiptHandler = handler @@ -38,7 +39,7 @@ func TestReceiptsHandlerChain(t *testing.T) { for nonce := 1; nonce < 21; nonce++ { var txError error var resp abci.ResponseDeliverTx - loomchain.NewSequence(util.PrefixKey([]byte("nonce"), addr1.Bytes())).Next(state) + sequence.NewSequence(util.PrefixKey([]byte("nonce"), addr1.Bytes())).Next(state) var txHash []byte if nonce%2 == 1 { // mock EVM transaction diff --git a/rpc/debug/jsonrpc_conversion.go b/rpc/debug/jsonrpc_conversion.go new file mode 100644 index 0000000000..c920cbb9fc --- /dev/null +++ b/rpc/debug/jsonrpc_conversion.go @@ -0,0 +1,29 @@ +// +build evm + +package debug + +import ( + "github.com/ethereum/go-ethereum/core/vm" + etheth "github.com/ethereum/go-ethereum/eth" +) + +type JsonTraceConfig struct { + DisableStorage bool `json:"disableStorage,omitempty"` + DisableMemory bool `json:"disableMemory,omitempty"` + DisableStack bool `json:"disableStack,omitempty"` + Tracer string `json:"tracer,omitempty"` + Timeout string `json:"address,omitempty"` +} + +func DecTraceConfig(jcfg JsonTraceConfig) etheth.TraceConfig { + return etheth.TraceConfig{ + LogConfig: &vm.LogConfig{ + DisableMemory: jcfg.DisableMemory, + DisableStack: jcfg.DisableStack, + DisableStorage: jcfg.DisableStorage, + }, + Tracer: &jcfg.Tracer, + Timeout: &jcfg.Timeout, + Reexec: nil, + } +} diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go new file mode 100644 index 0000000000..f107971772 --- /dev/null +++ b/rpc/debug/trace.go @@ -0,0 +1,35 @@ +// +build evm + +package debug + +import ( + "github.com/ethereum/go-ethereum/eth" + "github.com/pkg/errors" + + "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler/middleware" +) + +func TraceTransaction( + app middleware.InMemoryApp, + blockstore store.BlockStore, + blockNumber int64, + txIndex uint64, + config eth.TraceConfig, +) (interface{}, error) { + block, err := blockstore.GetBlockByHeight(&blockNumber) + if err != nil { + return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) + } + + for i := uint64(0); i < txIndex; i++ { + tx := block.Block.Data.Txs[i] + _, _ = app.ProcessTx(tx) + } + + result, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) + if err != nil { + return nil, err + } + return result, errors.New("not implemented") +} diff --git a/rpc/instrumenting.go b/rpc/instrumenting.go index 3a3df82e46..e464e2f2ce 100755 --- a/rpc/instrumenting.go +++ b/rpc/instrumenting.go @@ -8,6 +8,7 @@ import ( "github.com/gorilla/websocket" "github.com/loomnetwork/go-loom/plugin/types" "github.com/loomnetwork/loomchain/config" + "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" "github.com/loomnetwork/loomchain/vm" rpctypes "github.com/tendermint/tendermint/rpc/lib/types" @@ -569,3 +570,16 @@ func (m InstrumentingMiddleware) EthGetTransactionCount( resp, err = m.next.EthGetTransactionCount(local, block) return } + +func (m InstrumentingMiddleware) DebugTraceTransaction( + hash eth.Data, config debug.JsonTraceConfig, +) (resp interface{}, err error) { + defer func(begin time.Time) { + lvs := []string{"method", "DebugTraceTransaction", "error", fmt.Sprint(err != nil)} + m.requestCount.With(lvs...).Add(1) + m.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + + resp, err = m.next.DebugTraceTransaction(hash, config) + return +} diff --git a/rpc/mock_query_server.go b/rpc/mock_query_server.go index b6506afbbe..d0058b54a4 100644 --- a/rpc/mock_query_server.go +++ b/rpc/mock_query_server.go @@ -9,6 +9,7 @@ import ( "github.com/loomnetwork/go-loom/plugin/types" "github.com/loomnetwork/loomchain/config" + "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" "github.com/loomnetwork/loomchain/vm" ) @@ -368,3 +369,10 @@ func (m *MockQueryService) EvmUnSubscribe(id string) (bool, error) { m.MethodsCalled = append([]string{"EvmUnSubscribe"}, m.MethodsCalled...) return true, nil } + +func (m *MockQueryService) DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.MethodsCalled = append([]string{"DebugTraceTransaction"}, m.MethodsCalled...) + return nil, nil +} diff --git a/rpc/query_server.go b/rpc/query_server.go index d9825b452d..514599ff26 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -26,6 +26,7 @@ import ( "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/ethcoin" "github.com/loomnetwork/loomchain/config" "github.com/loomnetwork/loomchain/eth/polls" @@ -39,11 +40,13 @@ import ( "github.com/loomnetwork/loomchain/receipts/common" "github.com/loomnetwork/loomchain/registry" registryFac "github.com/loomnetwork/loomchain/registry/factory" + "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" blockindex "github.com/loomnetwork/loomchain/store/block_index" evmaux "github.com/loomnetwork/loomchain/store/evm_aux" + "github.com/loomnetwork/loomchain/txhandler/middleware" lvm "github.com/loomnetwork/loomchain/vm" ) @@ -61,6 +64,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State + InMemoryApp(uint64) middleware.InMemoryApp } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -131,7 +135,7 @@ type QueryServer struct { *evmaux.EvmAuxStore blockindex.BlockIndexStore EventStore store.EventStore - AuthCfg *auth.Config + AuthCfg *keys.Config } var _ QueryService = &QueryServer{} @@ -1108,6 +1112,23 @@ func (s *QueryServer) EthAccounts() ([]eth.Data, error) { return []eth.Data{}, nil } +func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) { + receipt, err := s.EthGetTransactionReceipt(hash) + if err != nil { + return nil, errors.Wrap(err, "cant find transaction matching hash") + } + blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) + if err != nil { + return nil, errors.Wrapf(err, "cant parse block number %v", receipt.BlockNumber) + } + txIndex, err := eth.DecQuantityToUint(receipt.TransactionIndex) + if err != nil { + return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) + } + cfg := debug.DecTraceConfig(config) + return debug.TraceTransaction(s.InMemoryApp(blockNumber), s.BlockStore, int64(blockNumber), txIndex, cfg) +} + func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { if nil != s.BlockIndexStore { return s.BlockIndexStore.GetBlockHeightByHash(hash) diff --git a/rpc/query_server_test.go b/rpc/query_server_test.go index 4ff1ab8128..cccb311245 100644 --- a/rpc/query_server_test.go +++ b/rpc/query_server_test.go @@ -102,6 +102,18 @@ func (s *stateProvider) ReadOnlyState() state.State { ) } +func (s *stateProvider) InMemoryApp() state.State { + return state.NewStoreState( + nil, + store.NewMemStore(), + abci.Header{ + ChainID: s.ChainID, + }, + nil, + nil, + ) +} + var testlog llog.TMLogger func TestQueryServer(t *testing.T) { diff --git a/rpc/query_service.go b/rpc/query_service.go index deca6d4663..d95bd06735 100755 --- a/rpc/query_service.go +++ b/rpc/query_service.go @@ -17,6 +17,7 @@ import ( "github.com/loomnetwork/loomchain/config" "github.com/loomnetwork/loomchain/eth/subs" "github.com/loomnetwork/loomchain/log" + "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" "github.com/loomnetwork/loomchain/vm" ) @@ -65,6 +66,9 @@ type QueryService interface { GetContractRecord(contractAddr string) (*types.ContractRecordResponse, error) + // debug transactions. + DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) + // deprecated function EvmTxReceipt(txHash []byte) ([]byte, error) GetEvmCode(contract string) ([]byte, error) @@ -191,6 +195,9 @@ func MakeEthQueryServiceHandler(svc QueryService, logger log.TMLogger, hub *Hub) routesJson["eth_getTransactionCount"] = eth.NewRPCFunc(svc.EthGetTransactionCount, "local,block") routesJson["eth_sendRawTransaction"] = eth.NewTendermintRPCFunc("eth_sendRawTransaction") + + routesJson["debug_traceTransaction"] = eth.NewRPCFunc(svc.DebugTraceTransaction, "hash,config") + RegisterRPCFuncs(wsmux, routesJson, logger, hub) mux := http.NewServeMux() diff --git a/store/splitstore.go b/store/splitstore.go new file mode 100644 index 0000000000..44f4671208 --- /dev/null +++ b/store/splitstore.go @@ -0,0 +1,78 @@ +package store + +import ( + "github.com/loomnetwork/go-loom/plugin" + "github.com/pkg/errors" +) + +type splitStore struct { + VersionedKVStore + diskStore VersionedKVStore + deleted map[string]bool +} + +func NewSplitStore(empty, full VersionedKVStore) VersionedKVStore { + return &splitStore{ + VersionedKVStore: empty, + diskStore: full, + deleted: make(map[string]bool), + } +} + +func (ss splitStore) Get(key []byte) []byte { + if ss.VersionedKVStore.Has(key) { + return ss.VersionedKVStore.Get(key) + } + if ss.deleted[string(key)] { + return nil + } + return ss.diskStore.Get(key) +} + +func (ss splitStore) Range(prefix []byte) plugin.RangeData { + resultRange := ss.VersionedKVStore.Range(prefix) + diskRange := ss.diskStore.Range(prefix) + for _, re := range diskRange { + if !ss.VersionedKVStore.Has(re.Key) && !ss.deleted[string(re.Key)] { + resultRange = append(resultRange, re) + } + } + return resultRange +} + +func (ss splitStore) Has(key []byte) bool { + if ss.VersionedKVStore.Has(key) { + return true + } + if ss.deleted[string(key)] { + return false + } + return ss.diskStore.Has(key) +} + +func (ss splitStore) Set(key, value []byte) { + ss.VersionedKVStore.Set(key, value) + ss.deleted[string(key)] = false +} + +func (ss splitStore) Delete(key []byte) { + ss.VersionedKVStore.Delete(key) + ss.deleted[string(key)] = true +} + +func (ss splitStore) Hash() []byte { + return nil +} +func (ss splitStore) Version() int64 { + return 0 +} +func (ss splitStore) SaveVersion() ([]byte, int64, error) { + return nil, 0, errors.New("not implemented") +} + +func (ss splitStore) Prune() error { + return errors.New("not implemented") +} +func (ss splitStore) GetSnapshot() Snapshot { + return nil +} diff --git a/throttle/contract_tx_limiter_middleware.go b/throttle/contract_tx_limiter_middleware.go index 6a7d4159c7..46736e514d 100644 --- a/throttle/contract_tx_limiter_middleware.go +++ b/throttle/contract_tx_limiter_middleware.go @@ -1,241 +1 @@ package throttle - -import ( - "fmt" - "time" - - "github.com/go-kit/kit/metrics" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/gogo/protobuf/proto" - "github.com/loomnetwork/go-loom" - udwtypes "github.com/loomnetwork/go-loom/builtin/types/user_deployer_whitelist" - "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" - udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" - appstate "github.com/loomnetwork/loomchain/state" - "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" - stdprometheus "github.com/prometheus/client_golang/prometheus" -) - -var ( - ErrTxLimitReached = errors.New("tx limit reached, try again later") - ErrContractNotWhitelisted = errors.New("contract not whitelisted") - ErrInactiveDeployer = errors.New("can't call contract belonging to inactive deployer") -) - -var ( - tierMapLoadLatency metrics.Histogram - contractTierMapLoadLatency metrics.Histogram -) - -func init() { - fieldKeys := []string{"method", "error"} - tierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "loomchain", - Subsystem: "contract_tx_limiter_middleware", - Name: "tier_map_load_latency", - Help: "Total time taken for Tier Map to Load in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, fieldKeys) - contractTierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "loomchain", - Subsystem: "contract_tx_limiter_middleware", - Name: "contract_tier_map_load_latency", - Help: "Total time taken for Contract Tier Map to Load in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, fieldKeys) -} - -type ContractTxLimiterConfig struct { - // Enables the middleware - Enabled bool - // Number of seconds each refresh lasts - ContractDataRefreshInterval int64 - TierDataRefreshInterval int64 -} - -func DefaultContractTxLimiterConfig() *ContractTxLimiterConfig { - return &ContractTxLimiterConfig{ - Enabled: false, - ContractDataRefreshInterval: 15 * 60, - TierDataRefreshInterval: 15 * 60, - } -} - -// Clone returns a deep clone of the config. -func (c *ContractTxLimiterConfig) Clone() *ContractTxLimiterConfig { - if c == nil { - return nil - } - clone := *c - return &clone -} - -type contractTxLimiter struct { - // contract_address to limiting parametres structure - contractToTierMap map[string]udw.TierID - inactiveDeployerContracts map[string]bool - contractDataLastUpdated int64 - // track of no. of txns in previous blocks per contract - contractStatsMap map[string]*contractStats - tierMap map[udw.TierID]udw.Tier - tierDataLastUpdated int64 -} - -type contractStats struct { - txn int64 - blockHeight int64 -} - -func (txl *contractTxLimiter) isAccountLimitReached(contractAddr loom.Address, curBlockHeight int64) bool { - blockTx, ok := txl.contractStatsMap[contractAddr.String()] - if !ok { - return false - } - // if execution reaches here => tierID and tier are valid - tierID := txl.contractToTierMap[contractAddr.String()] - tier := txl.tierMap[tierID] - if blockTx.blockHeight <= (curBlockHeight-int64(tier.BlockRange)) || int64(tier.MaxTxs) > blockTx.txn { - return false - } - return true -} - -func (txl *contractTxLimiter) updateState(contractAddr loom.Address, curBlockHeight int64) { - blockTx, ok := txl.contractStatsMap[contractAddr.String()] - tierID := txl.contractToTierMap[contractAddr.String()] - tier := txl.tierMap[tierID] - blockRange := int64(4096) // prevent divide by zero just in case tier doesn't have a range set - if tier.BlockRange > 0 { - blockRange = int64(tier.BlockRange) - } - if !ok || blockTx.blockHeight <= (curBlockHeight-blockRange) { - // resetting the blockHeight to lower bound of range instead of curblockheight - rangeStart := (((curBlockHeight - 1) / blockRange) * blockRange) + 1 - txl.contractStatsMap[contractAddr.String()] = &contractStats{1, rangeStart} - return - } - blockTx.txn++ -} - -func loadContractTierMap(ctx contractpb.StaticContext) (*udw.ContractInfo, error) { - var err error - defer func(begin time.Time) { - lvs := []string{"method", "loadContractTierMap", "error", fmt.Sprint(err != nil)} - contractTierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) - }(time.Now()) - contractInfo, err := udw.GetContractInfo(ctx) - return contractInfo, err -} - -func loadTierMap(ctx contractpb.StaticContext) (map[udwtypes.TierID]udwtypes.Tier, error) { - var err error - defer func(begin time.Time) { - lvs := []string{"method", "loadTierMap", "error", fmt.Sprint(err != nil)} - tierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) - }(time.Now()) - tierMap, err := udw.GetTierMap(ctx) - return tierMap, err -} - -// NewContractTxLimiterMiddleware creates a middleware function that limits how many call txs can be -// sent to an EVM contract within a pre-configured block range. -func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, - createUserDeployerWhitelistCtx func(state appstate.State) (contractpb.Context, error), -) loomchain.TxMiddlewareFunc { - txl := &contractTxLimiter{ - contractStatsMap: make(map[string]*contractStats), - } - return loomchain.TxMiddlewareFunc(func( - state appstate.State, - txBytes []byte, - next loomchain.TxHandlerFunc, - isCheckTx bool, - ) (res loomchain.TxHandlerResult, err error) { - if !isCheckTx { - return next(state, txBytes, isCheckTx) - } - var nonceTx auth.NonceTx - if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { - return res, errors.Wrap(err, "throttle: unwrap nonce Tx") - } - var tx loomchain.Transaction - if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { - return res, errors.New("throttle: unmarshal tx") - } - if tx.Id != callId { - return next(state, txBytes, isCheckTx) - } - var msg vm.MessageTx - if err := proto.Unmarshal(tx.Data, &msg); err != nil { - return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) - } - var msgTx vm.CallTx - if err := proto.Unmarshal(msg.Data, &msgTx); err != nil { - return res, errors.Wrapf(err, "unmarshal call tx %v", msg.Data) - } - if msgTx.VmType != vm.VMType_EVM { - return next(state, txBytes, isCheckTx) - } - if txl.inactiveDeployerContracts == nil || - txl.contractToTierMap == nil || - (txl.contractDataLastUpdated+cfg.ContractDataRefreshInterval) < time.Now().Unix() { - ctx, err := createUserDeployerWhitelistCtx(state) - if err != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - contractInfo, err := loadContractTierMap(ctx) - if err != nil { - return res, errors.Wrap(err, "throttle: contractInfo fetch") - } - - txl.contractDataLastUpdated = time.Now().Unix() - txl.contractToTierMap = contractInfo.ContractToTierMap - txl.inactiveDeployerContracts = contractInfo.InactiveDeployerContracts - // TxLimiter.contractDataLastUpdated will be updated after updating contractToTierMap - } - contractAddr := loom.UnmarshalAddressPB(msg.To) - // contracts which are deployed by deleted deployers should be throttled - if txl.inactiveDeployerContracts[contractAddr.String()] { - return res, ErrInactiveDeployer - } - // contracts the limiter doesn't know about shouldn't be throttled - contractTierID, ok := txl.contractToTierMap[contractAddr.String()] - if !ok { - return next(state, txBytes, isCheckTx) - } - if txl.tierMap == nil || - (txl.tierDataLastUpdated+cfg.TierDataRefreshInterval) < time.Now().Unix() { - ctx, er := createUserDeployerWhitelistCtx(state) - if er != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - txl.tierMap, err = loadTierMap(ctx) - if err != nil { - return res, errors.Wrap(err, "throttle: GetTierMap error") - } - txl.tierDataLastUpdated = time.Now().Unix() - } - // ensure that tier corresponding to contract available in tierMap - _, ok = txl.tierMap[contractTierID] - if !ok { - ctx, er := createUserDeployerWhitelistCtx(state) - if er != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - tierInfo, er := udw.GetTierInfo(ctx, contractTierID) - if er != nil { - return res, errors.Wrap(err, "throttle: getTierInfo error") - } - txl.tierMap[contractTierID] = tierInfo - } - if txl.isAccountLimitReached(contractAddr, state.Block().Height) { - return loomchain.TxHandlerResult{}, ErrTxLimitReached - } - txl.updateState(contractAddr, state.Block().Height) - - return next(state, txBytes, isCheckTx) - }) -} diff --git a/throttle/contract_tx_limiter_middleware_test.go b/throttle/contract_tx_limiter_middleware_test.go index b6818383fb..5ee8b57e48 100644 --- a/throttle/contract_tx_limiter_middleware_test.go +++ b/throttle/contract_tx_limiter_middleware_test.go @@ -9,13 +9,13 @@ import ( goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/builtin/plugins/coin" "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" @@ -113,9 +113,9 @@ func TestContractTxLimiterMiddleware(t *testing.T) { contractTxLimiterMiddleware.ProcessTx( state, txbytes, - func(_ appstate.State, txBytes []byte, isCheckTx bool) (res loomchain.TxHandlerResult, err error) { + func(_ appstate.State, txBytes []byte, isCheckTx bool) (res txhandler.TxHandlerResult, err error) { allowed = true - return loomchain.TxHandlerResult{}, nil + return txhandler.TxHandlerResult{}, nil }, true, ) diff --git a/throttle/karma-middleware.go b/throttle/karma-middleware.go index 485b32e4ad..ee0d69e73d 100644 --- a/throttle/karma-middleware.go +++ b/throttle/karma-middleware.go @@ -9,13 +9,15 @@ import ( lauth "github.com/loomnetwork/go-loom/auth" "github.com/loomnetwork/go-loom/common" "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/karma" "github.com/loomnetwork/loomchain/eth/utils" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" ) const karmaMiddlewareThrottleKey = "ThrottleTxMiddleWare" @@ -25,19 +27,19 @@ func GetKarmaMiddleWare( maxCallCount int64, sessionDuration int64, createKarmaContractCtx func(state appstate.State) (contractpb.Context, error), -) loomchain.TxMiddlewareFunc { +) txhandler.TxMiddlewareFunc { th := NewThrottle(sessionDuration, maxCallCount) - return loomchain.TxMiddlewareFunc(func( + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (res loomchain.TxHandlerResult, err error) { + ) (res txhandler.TxHandlerResult, err error) { if !karmaEnabled { return next(state, txBytes, isCheckTx) } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) if origin.IsEmpty() { return res, errors.New("throttle: transaction has no origin [get-karma]") } @@ -47,7 +49,7 @@ func GetKarmaMiddleWare( return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx loomchain.Transaction + var tx types.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } diff --git a/throttle/legacy_deployer_middleware.go b/throttle/legacy_deployer_middleware.go index 8c537c8ab0..9007612d48 100644 --- a/throttle/legacy_deployer_middleware.go +++ b/throttle/legacy_deployer_middleware.go @@ -5,24 +5,26 @@ import ( "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + + "github.com/loomnetwork/loomchain/auth/keys" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" ) // GetGoDeployTxMiddleWare creates middlware that only allows Go contract deployment tx to go through // if they originate from one of the allowed deployer accounts. This middleware has been superseded // by the DeployerWhitelist contract & middleware, though it's still in use on some clusters. -func GetGoDeployTxMiddleWare(allowedDeployers []loom.Address) loomchain.TxMiddlewareFunc { - return loomchain.TxMiddlewareFunc(func( +func GetGoDeployTxMiddleWare(allowedDeployers []loom.Address) txhandler.TxMiddlewareFunc { + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (res loomchain.TxHandlerResult, err error) { - var tx loomchain.Transaction + ) (res txhandler.TxHandlerResult, err error) { + var tx types.Transaction if err := proto.Unmarshal(txBytes, &tx); err != nil { return res, errors.Wrapf(err, "unmarshal tx %v", txBytes) } @@ -42,7 +44,7 @@ func GetGoDeployTxMiddleWare(allowedDeployers []loom.Address) loomchain.TxMiddle } if deployTx.VmType == vm.VMType_PLUGIN { - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) for _, allowed := range allowedDeployers { if 0 == origin.Compare(allowed) { return next(state, txBytes, isCheckTx) @@ -53,29 +55,3 @@ func GetGoDeployTxMiddleWare(allowedDeployers []loom.Address) loomchain.TxMiddle return next(state, txBytes, isCheckTx) }) } - -type GoContractDeployerWhitelistConfig struct { - Enabled bool - DeployerAddressList []string -} - -func DefaultGoContractDeployerWhitelistConfig() *GoContractDeployerWhitelistConfig { - return &GoContractDeployerWhitelistConfig{ - Enabled: false, - } -} - -func (c *GoContractDeployerWhitelistConfig) DeployerAddresses(chainId string) ([]loom.Address, error) { - deployerAddressList := make([]loom.Address, 0, len(c.DeployerAddressList)) - for _, addrStr := range c.DeployerAddressList { - addr, err := loom.ParseAddress(addrStr) - if err != nil { - addr, err = loom.ParseAddress(chainId + ":" + addrStr) - if err != nil { - return nil, errors.Wrapf(err, "parsing deploy address %s", addrStr) - } - } - deployerAddressList = append(deployerAddressList, addr) - } - return deployerAddressList, nil -} diff --git a/throttle/middleware_test_helper.go b/throttle/middleware_test_helper.go index ca6f23a840..a75d21e238 100644 --- a/throttle/middleware_test_helper.go +++ b/throttle/middleware_test_helper.go @@ -6,30 +6,32 @@ import ( "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/auth" - "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + loomAuth "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/eth/utils" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" ) var ( contract = loom.MustParseAddress("chain:0x9a1aC42a17AAD6Dbc6d21c162989d0f701074044") ) -func throttleMiddlewareHandler(ttm loomchain.TxMiddlewareFunc, state appstate.State, tx auth.SignedTx, ctx context.Context) (loomchain.TxHandlerResult, error) { +func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, tx auth.SignedTx, ctx context.Context) (txhandler.TxHandlerResult, error) { return ttm.ProcessTx( state.WithContext(ctx), tx.Inner, - func(state appstate.State, txBytes []byte, isCheckTx bool) (res loomchain.TxHandlerResult, err error) { + func(state appstate.State, txBytes []byte, isCheckTx bool) (res txhandler.TxHandlerResult, err error) { var nonceTx loomAuth.NonceTx if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx loomchain.Transaction + var tx types.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } @@ -66,7 +68,7 @@ func throttleMiddlewareHandler(ttm loomchain.TxMiddlewareFunc, state appstate.St }) } - return loomchain.TxHandlerResult{Data: data, Info: info}, err + return txhandler.TxHandlerResult{Data: data, Info: info}, err }, false, ) diff --git a/throttle/throttle.go b/throttle/throttle.go index 38f3886b52..38acfbf40f 100644 --- a/throttle/throttle.go +++ b/throttle/throttle.go @@ -14,7 +14,7 @@ import ( "github.com/ulule/limiter" "github.com/ulule/limiter/drivers/store/memory" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/karma" appstate "github.com/loomnetwork/loomchain/state" ) @@ -61,7 +61,7 @@ func (t *Throttle) getNewLimiter(ctx context.Context, limit int64) *limiter.Limi } func (t *Throttle) getLimiterFromPool(ctx context.Context, limit int64) *limiter.Limiter { - address := auth.Origin(ctx).String() + address := keys.Origin(ctx).String() _, ok := t.callLimiterPool[address] if !ok { t.callLimiterPool[address] = t.getNewLimiter(ctx, limit) @@ -75,7 +75,7 @@ func (t *Throttle) getLimiterFromPool(ctx context.Context, limit int64) *limiter } func (t *Throttle) getLimiterContext(ctx context.Context, nonce uint64, limit int64, txId uint32, key string) (limiter.Context, error) { - address := auth.Origin(ctx).String() + address := keys.Origin(ctx).String() if address == t.lastAddress && nonce == t.lastNonce && t.lastId == txId { return t.lastLimiterContext, nil } else { diff --git a/throttle/throttle_test.go b/throttle/throttle_test.go index 4730435cac..1237c58c1d 100644 --- a/throttle/throttle_test.go +++ b/throttle/throttle_test.go @@ -12,12 +12,12 @@ import ( goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" loomAuth "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/builtin/plugins/karma" "github.com/loomnetwork/loomchain/log" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" @@ -186,7 +186,7 @@ func mockSignedTx(t *testing.T, sequence uint64, id uint32, vmType vm.VMType, to require.NoError(t, err) } - tx, err := proto.Marshal(&loomchain.Transaction{ + tx, err := proto.Marshal(&txhandler.Transaction{ Id: id, Data: messageTx, }) diff --git a/throttle/tx_limiter_middleware.go b/throttle/tx_limiter_middleware.go index 70992cf006..46736e514d 100644 --- a/throttle/tx_limiter_middleware.go +++ b/throttle/tx_limiter_middleware.go @@ -1,94 +1 @@ package throttle - -import ( - "context" - "time" - - "github.com/loomnetwork/go-loom" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" - appstate "github.com/loomnetwork/loomchain/state" - "github.com/pkg/errors" - "github.com/ulule/limiter" - "github.com/ulule/limiter/drivers/store/memory" -) - -type TxLimiterConfig struct { - // Enables the tx limiter middleware - Enabled bool - // Number of seconds each session lasts - SessionDuration int64 - // Maximum number of txs that should be allowed per session - MaxTxsPerSession int64 -} - -func DefaultTxLimiterConfig() *TxLimiterConfig { - return &TxLimiterConfig{ - SessionDuration: 60, - MaxTxsPerSession: 60, - } -} - -// Clone returns a deep clone of the config. -func (c *TxLimiterConfig) Clone() *TxLimiterConfig { - if c == nil { - return nil - } - clone := *c - return &clone -} - -type txLimiter struct { - *limiter.Limiter -} - -func newTxLimiter(cfg *TxLimiterConfig) *txLimiter { - return &txLimiter{ - Limiter: limiter.New( - memory.NewStore(), - limiter.Rate{ - Period: time.Duration(cfg.SessionDuration) * time.Second, - Limit: cfg.MaxTxsPerSession, - }, - ), - } -} - -func (txl *txLimiter) isAccountLimitReached(account loom.Address) bool { - lmtCtx, err := txl.Limiter.Get(context.TODO(), account.String()) - // Doesn't look like the current implementation of the limit with the in-memory store will ever - // return an error anyway. - if err != nil { - panic(err) - } - return lmtCtx.Reached -} - -// NewTxLimiterMiddleware creates middleware that throttles txs (all types) in CheckTx, the rate -// can be configured in loom.yml. Since this middleware only runs in CheckTx the rate limit can -// differ between nodes on the same cluster, and private nodes don't really need to run the rate -// limiter at all. -func NewTxLimiterMiddleware(cfg *TxLimiterConfig) loomchain.TxMiddlewareFunc { - txl := newTxLimiter(cfg) - return loomchain.TxMiddlewareFunc(func( - state appstate.State, - txBytes []byte, - next loomchain.TxHandlerFunc, - isCheckTx bool, - ) (loomchain.TxHandlerResult, error) { - if !isCheckTx { - return next(state, txBytes, isCheckTx) - } - - origin := auth.Origin(state.Context()) - if origin.IsEmpty() { - return loomchain.TxHandlerResult{}, errors.New("throttle: transaction has no origin [get-karma]") - } - - if txl.isAccountLimitReached(origin) { - return loomchain.TxHandlerResult{}, errors.New("tx limit reached, try again later") - } - - return next(state, txBytes, isCheckTx) - }) -} diff --git a/tx_handler/migration_tx_handler.go b/tx_handler/migration_tx_handler.go index 759d3ad3a1..922f087934 100644 --- a/tx_handler/migration_tx_handler.go +++ b/tx_handler/migration_tx_handler.go @@ -5,18 +5,19 @@ import ( "encoding/binary" "fmt" - proto "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/proto" "github.com/pkg/errors" - loom "github.com/loomnetwork/go-loom" + "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/util" goloomvm "github.com/loomnetwork/go-loom/vm" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/migrations" registry "github.com/loomnetwork/loomchain/registry/factory" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" ) @@ -48,8 +49,8 @@ func (h *MigrationTxHandler) ProcessTx( state appstate.State, txBytes []byte, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult if !state.FeatureEnabled(features.MigrationTxFeature, false) { return r, fmt.Errorf("MigrationTx feature hasn't been enabled") @@ -61,7 +62,7 @@ func (h *MigrationTxHandler) ProcessTx( return r, err } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) caller := loom.UnmarshalAddressPB(msg.From) if caller.Compare(origin) != 0 { diff --git a/middleware.go b/txhandler/middleware.go similarity index 99% rename from middleware.go rename to txhandler/middleware.go index d5325450de..ec0d60cd34 100644 --- a/middleware.go +++ b/txhandler/middleware.go @@ -1,4 +1,4 @@ -package loomchain +package txhandler import ( "encoding/base64" diff --git a/throttle/deployer-middleware.go b/txhandler/middleware/deployer-middleware.go similarity index 86% rename from throttle/deployer-middleware.go rename to txhandler/middleware/deployer-middleware.go index e93bf3dae1..73f9da6ca1 100644 --- a/throttle/deployer-middleware.go +++ b/txhandler/middleware/deployer-middleware.go @@ -1,18 +1,27 @@ -package throttle +package middleware import ( "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" + "github.com/loomnetwork/go-loom/auth" "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + + "github.com/loomnetwork/loomchain/auth/keys" dw "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" +) + +const ( + deployId = uint32(1) + callId = uint32(2) + migrationId = uint32(3) ) var ( @@ -27,12 +36,12 @@ var ( // Records deploymentAddress and vmType func NewEVMDeployRecorderPostCommitMiddleware( createDeployerWhitelistCtx func(state appstate.State) (contractpb.Context, error), -) (loomchain.PostCommitMiddleware, error) { - return loomchain.PostCommitMiddlewareFunc(func( +) (txhandler.PostCommitMiddleware, error) { + return txhandler.PostCommitMiddlewareFunc(func( state appstate.State, txBytes []byte, - res loomchain.TxHandlerResult, - next loomchain.PostCommitHandler, + res txhandler.TxHandlerResult, + next txhandler.PostCommitHandler, isCheckTx bool, ) error { if !state.FeatureEnabled(features.UserDeployerWhitelistFeature, false) { @@ -54,7 +63,7 @@ func NewEVMDeployRecorderPostCommitMiddleware( return errors.Wrapf(err, "unmarshal deploy response %v", res.Data) } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) ctx, err := createDeployerWhitelistCtx(state) if err != nil { return err @@ -70,13 +79,13 @@ func NewEVMDeployRecorderPostCommitMiddleware( func NewDeployerWhitelistMiddleware( createDeployerWhitelistCtx func(state appstate.State) (contractpb.Context, error), -) (loomchain.TxMiddlewareFunc, error) { - return loomchain.TxMiddlewareFunc(func( +) (txhandler.TxMiddlewareFunc, error) { + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (res loomchain.TxHandlerResult, err error) { + ) (res txhandler.TxHandlerResult, err error) { if !state.FeatureEnabled(features.DeployerWhitelistFeature, false) { return next(state, txBytes, isCheckTx) @@ -87,7 +96,7 @@ func NewDeployerWhitelistMiddleware( return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx loomchain.Transaction + var tx types.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } @@ -109,7 +118,7 @@ func NewDeployerWhitelistMiddleware( } if deployTx.VmType == vm.VMType_PLUGIN { - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) ctx, err := createDeployerWhitelistCtx(state) if err != nil { return res, err @@ -118,7 +127,7 @@ func NewDeployerWhitelistMiddleware( return res, err } } else if deployTx.VmType == vm.VMType_EVM { - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) ctx, err := createDeployerWhitelistCtx(state) if err != nil { return res, err @@ -130,7 +139,7 @@ func NewDeployerWhitelistMiddleware( } else if tx.Id == migrationId { // Process migrationTx, checking for permission to migrate contract - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) ctx, err := createDeployerWhitelistCtx(state) if err != nil { return res, err diff --git a/throttle/deployer-middleware_test.go b/txhandler/middleware/deployer-middleware_test.go similarity index 91% rename from throttle/deployer-middleware_test.go rename to txhandler/middleware/deployer-middleware_test.go index c0a15cbec3..92072d3df7 100644 --- a/throttle/deployer-middleware_test.go +++ b/txhandler/middleware/deployer-middleware_test.go @@ -1,4 +1,4 @@ -package throttle +package middleware import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - loomAuth "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" dw "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" @@ -41,8 +41,8 @@ func TestDeployerWhitelistMiddleware(t *testing.T) { Owner: owner.MarshalPB(), })) - guestCtx := context.WithValue(state.Context(), loomAuth.ContextKeyOrigin, guest) - ownerCtx := context.WithValue(state.Context(), loomAuth.ContextKeyOrigin, owner) + guestCtx := context.WithValue(state.Context(), keys.ContextKeyOrigin, guest) + ownerCtx := context.WithValue(state.Context(), keys.ContextKeyOrigin, owner) dwMiddleware, err := NewDeployerWhitelistMiddleware( func(_ appstate.State) (contractpb.Context, error) { diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go new file mode 100644 index 0000000000..cdf9a151f1 --- /dev/null +++ b/txhandler/middleware/memory_app.go @@ -0,0 +1,80 @@ +package middleware + +import ( + "context" + + "github.com/ethereum/go-ethereum/eth" + "github.com/loomnetwork/go-loom/builtin/types/chainconfig" + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" +) + +type InMemoryApp interface { + ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) + TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, error) +} + +type inMemoryApp struct { + LastBlockHeader abci.Header + CurBlockHeader abci.Header + CurBlockHash []byte + Store store.VersionedKVStore + TxHandler txhandler.TxHandler + ReceiptsVersion int32 + GetValidatorSet state.GetValidatorSet + config *chainconfig.Config +} + +func NewInMemoryApp( + lastBlockHeader abci.Header, + curBlockHeader abci.Header, + curBlockHash []byte, + store store.VersionedKVStore, + txHandler txhandler.TxHandler, + receiptsVersion int32, + getValidatorSet state.GetValidatorSet, + config *chainconfig.Config, +) InMemoryApp { + return &inMemoryApp{ + LastBlockHeader: lastBlockHeader, + CurBlockHeader: curBlockHeader, + CurBlockHash: curBlockHash, + Store: store, + TxHandler: txHandler, + ReceiptsVersion: receiptsVersion, + GetValidatorSet: getValidatorSet, + config: config, + } +} + +func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) { + return ma.processTx(txBytes, ma.TxHandler) +} + +func (ma *inMemoryApp) processTx(txBytes []byte, txHandler txhandler.TxHandler) (txhandler.TxHandlerResult, error) { + splitStoreTx := store.WrapAtomic(ma.Store).BeginTx() + defer splitStoreTx.Rollback() + storeState := state.NewStoreState( + context.Background(), + splitStoreTx, + ma.CurBlockHeader, + ma.CurBlockHash, + ma.GetValidatorSet, + ).WithOnChainConfig(ma.config) + return txHandler.ProcessTx(storeState, txBytes, false) +} + +func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, error) { + traceTxHandle, err := TraceTxHandle(traceCfg) + if err != nil { + return txhandler.TxHandlerResult{}, err + } + return ma.processTx(txBytes, traceTxHandle) +} + +func TraceTxHandle(traceCfg eth.TraceConfig) (txhandler.TxHandler, error) { + return nil, nil +} diff --git a/router.go b/txhandler/middleware/router.go similarity index 76% rename from router.go rename to txhandler/middleware/router.go index d8dd531b0c..8ca1008b5c 100644 --- a/router.go +++ b/txhandler/middleware/router.go @@ -1,4 +1,4 @@ -package loomchain +package middleware import ( "github.com/gogo/protobuf/proto" @@ -6,27 +6,26 @@ import ( "github.com/loomnetwork/go-loom/types" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" ) -type Transaction = types.Transaction - type TxRouter struct { deliverTxRoutes map[uint32]RouteHandler checkTxRoutes map[uint32]RouteHandler } -type RouteHandler func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) +type RouteHandler func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) type RouteConditionFunc func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) bool -var GeneratePassthroughRouteHandler = func(txHandler TxHandler) RouteHandler { - return func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { +var GeneratePassthroughRouteHandler = func(txHandler txhandler.TxHandler) RouteHandler { + return func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txHandler.ProcessTx(state, txBytes, isCheckTx) } } -func GenerateConditionalRouteHandler(conditionFn RouteConditionFunc, onTrue TxHandler, onFalse TxHandler) RouteHandler { - return RouteHandler(func(txId uint32, state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { +func GenerateConditionalRouteHandler(conditionFn RouteConditionFunc, onTrue txhandler.TxHandler, onFalse txhandler.TxHandler) RouteHandler { + return RouteHandler(func(txId uint32, state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { if conditionFn(txId, state, txBytes, isCheckTx) { return onTrue.ProcessTx(state, txBytes, isCheckTx) } @@ -57,10 +56,10 @@ func (r *TxRouter) HandleCheckTx(txID uint32, handler RouteHandler) { r.checkTxRoutes[txID] = handler } -func (r *TxRouter) ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { - var res TxHandlerResult +func (r *TxRouter) ProcessTx(state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { + var res txhandler.TxHandlerResult - var tx Transaction + var tx types.Transaction err := proto.Unmarshal(txBytes, &tx) if err != nil { return res, err diff --git a/txhandler/middleware/throttle.go b/txhandler/middleware/throttle.go new file mode 100644 index 0000000000..9f9b8bed51 --- /dev/null +++ b/txhandler/middleware/throttle.go @@ -0,0 +1,276 @@ +package middleware + +import ( + "context" + "fmt" + "time" + + "github.com/go-kit/kit/metrics" + kitprometheus "github.com/go-kit/kit/metrics/prometheus" + "github.com/gogo/protobuf/proto" + "github.com/loomnetwork/go-loom" + "github.com/loomnetwork/go-loom/plugin/contractpb" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/ulule/limiter" + "github.com/ulule/limiter/drivers/store/memory" + + udwtypes "github.com/loomnetwork/go-loom/builtin/types/user_deployer_whitelist" + + "github.com/loomnetwork/go-loom/auth" + + "github.com/loomnetwork/loomchain/auth/keys" + udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" + "github.com/loomnetwork/loomchain/config" + "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/vm" +) + +var ( + tierMapLoadLatency metrics.Histogram + contractTierMapLoadLatency metrics.Histogram + ErrInactiveDeployer = errors.New("can't call contract belonging to inactive deployer") + ErrTxLimitReached = errors.New("tx limit reached, try again later") +) + +type txLimiter struct { + *limiter.Limiter +} + +func newTxLimiter(cfg *config.TxLimiterConfig) *txLimiter { + return &txLimiter{ + Limiter: limiter.New( + memory.NewStore(), + limiter.Rate{ + Period: time.Duration(cfg.SessionDuration) * time.Second, + Limit: cfg.MaxTxsPerSession, + }, + ), + } +} + +func (txl *txLimiter) isAccountLimitReached(account loom.Address) bool { + lmtCtx, err := txl.Limiter.Get(context.TODO(), account.String()) + // Doesn't look like the current implementation of the limit with the in-memory store will ever + // return an error anyway. + if err != nil { + panic(err) + } + return lmtCtx.Reached +} + +// NewTxLimiterMiddleware creates middleware that throttles txs (all types) in CheckTx, the rate +// can be configured in loom.yml. Since this middleware only runs in CheckTx the rate limit can +// differ between nodes on the same cluster, and private nodes don't really need to run the rate +// limiter at all. +func NewTxLimiterMiddleware(cfg *config.TxLimiterConfig) txhandler.TxMiddlewareFunc { + txl := newTxLimiter(cfg) + return txhandler.TxMiddlewareFunc(func( + _state state.State, + txBytes []byte, + next txhandler.TxHandlerFunc, + isCheckTx bool, + ) (txhandler.TxHandlerResult, error) { + if !isCheckTx { + return next(_state, txBytes, isCheckTx) + } + + origin := keys.Origin(_state.Context()) + if origin.IsEmpty() { + return txhandler.TxHandlerResult{}, errors.New("throttle: transaction has no origin [get-karma]") + } + + if txl.isAccountLimitReached(origin) { + return txhandler.TxHandlerResult{}, errors.New("tx limit reached, try again later") + } + + return next(_state, txBytes, isCheckTx) + }) +} + +func init() { + fieldKeys := []string{"method", "error"} + tierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ + Namespace: "loomchain", + Subsystem: "contract_tx_limiter_middleware", + Name: "tier_map_load_latency", + Help: "Total time taken for Tier Map to Load in seconds.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, fieldKeys) + contractTierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ + Namespace: "loomchain", + Subsystem: "contract_tx_limiter_middleware", + Name: "contract_tier_map_load_latency", + Help: "Total time taken for Contract Tier Map to Load in seconds.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, fieldKeys) +} + +type contractTxLimiter struct { + // contract_address to limiting parametres structure + contractToTierMap map[string]udw.TierID + inactiveDeployerContracts map[string]bool + contractDataLastUpdated int64 + // track of no. of txns in previous blocks per contract + contractStatsMap map[string]*contractStats + tierMap map[udw.TierID]udw.Tier + tierDataLastUpdated int64 +} + +type contractStats struct { + txn int64 + blockHeight int64 +} + +func (txl *contractTxLimiter) isAccountLimitReached(contractAddr loom.Address, curBlockHeight int64) bool { + blockTx, ok := txl.contractStatsMap[contractAddr.String()] + if !ok { + return false + } + // if execution reaches here => tierID and tier are valid + tierID := txl.contractToTierMap[contractAddr.String()] + tier := txl.tierMap[tierID] + if blockTx.blockHeight <= (curBlockHeight-int64(tier.BlockRange)) || int64(tier.MaxTxs) > blockTx.txn { + return false + } + return true +} + +func (txl *contractTxLimiter) updateState(contractAddr loom.Address, curBlockHeight int64) { + blockTx, ok := txl.contractStatsMap[contractAddr.String()] + tierID := txl.contractToTierMap[contractAddr.String()] + tier := txl.tierMap[tierID] + blockRange := int64(4096) // prevent divide by zero just in case tier doesn't have a range set + if tier.BlockRange > 0 { + blockRange = int64(tier.BlockRange) + } + if !ok || blockTx.blockHeight <= (curBlockHeight-blockRange) { + // resetting the blockHeight to lower bound of range instead of curblockheight + rangeStart := (((curBlockHeight - 1) / blockRange) * blockRange) + 1 + txl.contractStatsMap[contractAddr.String()] = &contractStats{1, rangeStart} + return + } + blockTx.txn++ +} + +func loadContractTierMap(ctx contractpb.StaticContext) (*udw.ContractInfo, error) { + var err error + defer func(begin time.Time) { + lvs := []string{"method", "loadContractTierMap", "error", fmt.Sprint(err != nil)} + contractTierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + contractInfo, err := udw.GetContractInfo(ctx) + return contractInfo, err +} + +func loadTierMap(ctx contractpb.StaticContext) (map[udwtypes.TierID]udwtypes.Tier, error) { + var err error + defer func(begin time.Time) { + lvs := []string{"method", "loadTierMap", "error", fmt.Sprint(err != nil)} + tierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + tierMap, err := udw.GetTierMap(ctx) + return tierMap, err +} + +// NewContractTxLimiterMiddleware creates a middleware function that limits how many call txs can be +// sent to an EVM contract within a pre-configured block range. +func NewContractTxLimiterMiddleware(cfg *config.ContractTxLimiterConfig, + createUserDeployerWhitelistCtx func(_state state.State) (contractpb.Context, error), +) txhandler.TxMiddlewareFunc { + txl := &contractTxLimiter{ + contractStatsMap: make(map[string]*contractStats), + } + return txhandler.TxMiddlewareFunc(func( + _state state.State, + txBytes []byte, + next txhandler.TxHandlerFunc, + isCheckTx bool, + ) (res txhandler.TxHandlerResult, err error) { + if !isCheckTx { + return next(_state, txBytes, isCheckTx) + } + var nonceTx auth.NonceTx + if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { + return res, errors.Wrap(err, "throttle: unwrap nonce Tx") + } + var tx types.Transaction + if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { + return res, errors.New("throttle: unmarshal tx") + } + if tx.Id != callId { + return next(_state, txBytes, isCheckTx) + } + var msg vm.MessageTx + if err := proto.Unmarshal(tx.Data, &msg); err != nil { + return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) + } + var msgTx vm.CallTx + if err := proto.Unmarshal(msg.Data, &msgTx); err != nil { + return res, errors.Wrapf(err, "unmarshal call tx %v", msg.Data) + } + if msgTx.VmType != vm.VMType_EVM { + return next(_state, txBytes, isCheckTx) + } + if txl.inactiveDeployerContracts == nil || + txl.contractToTierMap == nil || + (txl.contractDataLastUpdated+cfg.ContractDataRefreshInterval) < time.Now().Unix() { + ctx, err := createUserDeployerWhitelistCtx(_state) + if err != nil { + return res, errors.Wrap(err, "throttle: context creation") + } + contractInfo, err := loadContractTierMap(ctx) + if err != nil { + return res, errors.Wrap(err, "throttle: contractInfo fetch") + } + + txl.contractDataLastUpdated = time.Now().Unix() + txl.contractToTierMap = contractInfo.ContractToTierMap + txl.inactiveDeployerContracts = contractInfo.InactiveDeployerContracts + // TxLimiter.contractDataLastUpdated will be updated after updating contractToTierMap + } + contractAddr := loom.UnmarshalAddressPB(msg.To) + // contracts which are deployed by deleted deployers should be throttled + if txl.inactiveDeployerContracts[contractAddr.String()] { + return res, ErrInactiveDeployer + } + // contracts the limiter doesn't know about shouldn't be throttled + contractTierID, ok := txl.contractToTierMap[contractAddr.String()] + if !ok { + return next(_state, txBytes, isCheckTx) + } + if txl.tierMap == nil || + (txl.tierDataLastUpdated+cfg.TierDataRefreshInterval) < time.Now().Unix() { + ctx, er := createUserDeployerWhitelistCtx(_state) + if er != nil { + return res, errors.Wrap(err, "throttle: context creation") + } + txl.tierMap, err = loadTierMap(ctx) + if err != nil { + return res, errors.Wrap(err, "throttle: GetTierMap error") + } + txl.tierDataLastUpdated = time.Now().Unix() + } + // ensure that tier corresponding to contract available in tierMap + _, ok = txl.tierMap[contractTierID] + if !ok { + ctx, er := createUserDeployerWhitelistCtx(_state) + if er != nil { + return res, errors.Wrap(err, "throttle: context creation") + } + tierInfo, er := udw.GetTierInfo(ctx, contractTierID) + if er != nil { + return res, errors.Wrap(err, "throttle: getTierInfo error") + } + txl.tierMap[contractTierID] = tierInfo + } + if txl.isAccountLimitReached(contractAddr, _state.Block().Height) { + return txhandler.TxHandlerResult{}, ErrTxLimitReached + } + txl.updateState(contractAddr, _state.Block().Height) + + return next(_state, txBytes, isCheckTx) + }) +} diff --git a/middleware_test.go b/txhandler/middleware_test.go similarity index 99% rename from middleware_test.go rename to txhandler/middleware_test.go index 8e98d35b3c..a574660964 100644 --- a/middleware_test.go +++ b/txhandler/middleware_test.go @@ -1,4 +1,4 @@ -package loomchain +package txhandler import ( "testing" diff --git a/txhandler/txhandler.go b/txhandler/txhandler.go new file mode 100644 index 0000000000..1af2bd32b1 --- /dev/null +++ b/txhandler/txhandler.go @@ -0,0 +1,27 @@ +package txhandler + +import ( + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/common" + + "github.com/loomnetwork/loomchain/state" +) + +type TxHandler interface { + ProcessTx(state.State, []byte, bool) (TxHandlerResult, error) +} + +type TxHandlerFunc func(state.State, []byte, bool) (TxHandlerResult, error) + +type TxHandlerResult struct { + Data []byte + ValidatorUpdates []abci.Validator + Info string + // Tags to associate with the tx that produced this result. Tags can be used to filter txs + // via the ABCI query interface (see https://godoc.org/github.com/tendermint/tendermint/libs/pubsub/query) + Tags []common.KVPair +} + +func (f TxHandlerFunc) ProcessTx(s state.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { + return f(s, txBytes, isCheckTx) +} diff --git a/vm/handler.go b/vm/handler.go index b9ec579307..c2af845c62 100644 --- a/vm/handler.go +++ b/vm/handler.go @@ -8,12 +8,12 @@ import ( "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" registry "github.com/loomnetwork/loomchain/registry/factory" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" ) type DeployTxHandler struct { @@ -26,8 +26,8 @@ func (h *DeployTxHandler) ProcessTx( state appstate.State, txBytes []byte, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult var msg MessageTx err := proto.Unmarshal(txBytes, &msg) @@ -35,7 +35,7 @@ func (h *DeployTxHandler) ProcessTx( return r, err } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) caller := loom.UnmarshalAddressPB(msg.From) if caller.Compare(origin) != 0 { @@ -108,8 +108,8 @@ func (h *CallTxHandler) ProcessTx( txBytes []byte, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult var msg MessageTx err := proto.Unmarshal(txBytes, &msg) @@ -117,7 +117,7 @@ func (h *CallTxHandler) ProcessTx( return r, err } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) caller := loom.UnmarshalAddressPB(msg.From) addr := loom.UnmarshalAddressPB(msg.To) From 414db2435944cb8810a237673fd2a7baf25c16a4 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 24 Sep 2019 13:47:53 +0100 Subject: [PATCH 02/49] implement --- app.go | 4 +- .../sample_go_contract_test.go | 2 +- cmd/loom/db/evm.go | 4 +- cmd/loom/loom.go | 64 ++--- evm/evm.go | 40 ++- evm/loomevm.go | 23 +- evm/noevm.go | 11 +- integration_tests/ethcoin_evm_test.go | 6 +- plugin/fake_context.go | 4 +- plugin/vm.go | 4 +- plugin/vm_test.go | 6 +- rpc/debug/trace.go | 23 +- rpc/query_server.go | 8 +- txhandler/factory/handler_factory.go | 248 ++++++++++++++++++ txhandler/middleware/memory_app.go | 74 ++++-- txhandler/txhandler.go | 5 + 16 files changed, 404 insertions(+), 122 deletions(-) create mode 100644 txhandler/factory/handler_factory.go diff --git a/app.go b/app.go index 2e05308875..b05a20a230 100644 --- a/app.go +++ b/app.go @@ -58,6 +58,7 @@ type Application struct { QueryHandler EventHandler ReceiptHandlerProvider + txhandler.TxHandlerFactory EvmAuxStore *evmaux.EvmAuxStore blockindex.BlockIndexStore CreateValidatorManager ValidatorsManagerFactoryFunc @@ -606,8 +607,6 @@ func (a *Application) ReadOnlyState() appstate.State { } func (a *Application) InMemoryApp(blockNumber uint64) middleware.InMemoryApp { - // TODO: the store snapshot should be created atomically, otherwise the block header might - // not match the state... need to figure out why this hasn't spectacularly failed already return middleware.NewInMemoryApp( a.lastBlockHeader, a.curBlockHeader, @@ -617,5 +616,6 @@ func (a *Application) InMemoryApp(blockNumber uint64) middleware.InMemoryApp { a.ReceiptsVersion, a.GetValidatorSet, a.config, + a.TxHandlerFactory, ) } diff --git a/builtin/plugins/sample_go_contract/sample_go_contract_test.go b/builtin/plugins/sample_go_contract/sample_go_contract_test.go index b125518992..5defad796f 100644 --- a/builtin/plugins/sample_go_contract/sample_go_contract_test.go +++ b/builtin/plugins/sample_go_contract/sample_go_contract_test.go @@ -59,7 +59,7 @@ func deployContractToEVM(ctx *plugin.FakeContextWithEVM, filename string, caller byteCode := common.FromHex(string(hexByteCode)) byteCode, err = hex.DecodeString(string(hexByteCode)) - vm := evm.NewLoomVm(ctx.State, nil, nil, nil, false) + vm := evm.NewLoomVm(ctx.State, nil, nil, false, nil) _, contractAddr, err = vm.Create(caller, byteCode, loom.NewBigUIntFromInt(0)) if err != nil { return contractAddr, err diff --git a/cmd/loom/db/evm.go b/cmd/loom/db/evm.go index 9c27b34f65..fa2aa57a75 100644 --- a/cmd/loom/db/evm.go +++ b/cmd/loom/db/evm.go @@ -111,7 +111,7 @@ func newDumpEVMStateCommand() *cobra.Command { } } - vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false) + vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false, nil) if err != nil { return err } @@ -224,7 +224,7 @@ func newDumpEVMStateMultiWriterAppStoreCommand() *cobra.Command { } } - vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false) + vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false, nil) if err != nil { return err } diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index c0bfa06bbd..1ac3c021aa 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -43,6 +43,7 @@ import ( "github.com/loomnetwork/loomchain/throttle" "github.com/loomnetwork/loomchain/tx_handler" "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/txhandler/factory" "github.com/loomnetwork/loomchain/txhandler/middleware" "github.com/pkg/errors" @@ -824,7 +825,7 @@ func loadApp( return nil, err } } - return evm.NewLoomVm(state, eventHandler, receiptHandlerProvider.Writer(), createABM, cfg.EVMDebugEnabled), nil + return evm.NewLoomVm(state, receiptHandlerProvider.Writer(), createABM, cfg.EVMDebugEnabled, nil), nil }) } evm.LogEthDbBatch = cfg.LogEthDbBatch @@ -849,7 +850,7 @@ func loadApp( err := deployContract( state, contractCfg, - vmManager, + *vmManager, rootAddr, registry, logger, @@ -862,7 +863,7 @@ func loadApp( return nil } - createKarmaContractCtx := getContractCtx("karma", vmManager) + createKarmaContractCtx := getContractCtx("karma", *vmManager) createContractUpkeepHandler := func(state appstate.State) (loomchain.KarmaHandler, error) { // TODO: This setting should be part of the config stored within the Karma contract itself, @@ -884,7 +885,7 @@ func loadApp( getValidatorSet := func(state appstate.State) (loom.ValidatorSet, error) { if cfg.DPOSVersion == 3 || state.FeatureEnabled(features.DPOSVersion3Feature, false) { - createDPOSV3Ctx := getContractCtx("dposV3", vmManager) + createDPOSV3Ctx := getContractCtx("dposV3", *vmManager) dposV3Ctx, err := createDPOSV3Ctx(state) if err != nil { return nil, err @@ -895,7 +896,7 @@ func loadApp( } return loom.NewValidatorSet(validators...), nil } else if cfg.DPOSVersion == 2 { - createDPOSV2Ctx := getContractCtx("dposV2", vmManager) + createDPOSV2Ctx := getContractCtx("dposV2", *vmManager) dposV2Ctx, err := createDPOSV2Ctx(state) if err != nil { return nil, err @@ -970,17 +971,22 @@ func loadApp( return nil, err } */ - postCommitMiddlewares, err := postCommitMiddleWAre(cfg, vmManager) - if err != nil { - return nil, err - } + //postCommitMiddlewares, err := postCommitMiddleWAre(*cfg, *vmManager) + //if err != nil { + // return nil, err + //} //txHandler, err := middleware.AppTxHandler(cfg, vmManager, createRegistry, chainID, appStore) //if err != nil { // return nil, err //} - txMiddleware, err := txMiddleWare(cfg, vmManager, chainID, appStore) + //txMiddleware, err := txMiddleWare(*cfg, *vmManager, chainID, appStore) + //if err != nil { + // return nil, err + //} + txHandlerFactory := factory.NewTxHandlerFactory(*cfg, vmManager, chainID, appStore) + chainTxHandler, err := txHandlerFactory.TxHandler(nil) if err != nil { return nil, err } @@ -988,11 +994,13 @@ func loadApp( return &loomchain.Application{ Store: appStore, Init: init, - TxHandler: txhandler.MiddlewareTxHandler( - txMiddleware, - router(cfg, vmManager, createRegistry), - postCommitMiddlewares, - ), + //TxHandler: txhandler.MiddlewareTxHandler( + // txMiddleware, + // router(*cfg, *vmManager, createRegistry), + // postCommitMiddlewares, + //), + TxHandler: chainTxHandler, + TxHandlerFactory: txHandlerFactory, BlockIndexStore: blockIndexStore, EventHandler: eventHandler, ReceiptHandlerProvider: receiptHandlerProvider, @@ -1009,7 +1017,7 @@ func loadApp( func deployContract( state appstate.State, contractCfg config.ContractConfig, - vmManager *vm.Manager, + vmManager vm.Manager, rootAddr loom.Address, registry regcommon.Registry, logger log.TMLogger, @@ -1051,8 +1059,8 @@ func deployContract( } func txMiddleWare( - cfg *config.Config, - vmManager *vm.Manager, + cfg config.Config, + vmManager vm.Manager, chainID string, appStore store.VersionedKVStore, ) ([]txhandler.TxMiddleware, error) { @@ -1111,8 +1119,8 @@ func txMiddleWare( } func router( - cfg *config.Config, - vmManager *vm.Manager, + cfg config.Config, + vmManager vm.Manager, createRegistry registry.RegistryFactoryFunc, ) txhandler.TxHandler { router := middleware.NewTxRouter() @@ -1150,17 +1158,17 @@ func router( } deployTxHandler := &vm.DeployTxHandler{ - Manager: vmManager, + Manager: &vmManager, CreateRegistry: createRegistry, AllowNamedEVMContracts: cfg.AllowNamedEvmContracts, } callTxHandler := &vm.CallTxHandler{ - Manager: vmManager, + Manager: &vmManager, } migrationTxHandler := &tx_handler.MigrationTxHandler{ - Manager: vmManager, + Manager: &vmManager, CreateRegistry: createRegistry, Migrations: map[int32]tx_handler.MigrationFunc{ 1: migrations.DPOSv3Migration, @@ -1181,7 +1189,7 @@ func router( return router } -func postCommitMiddleWAre(cfg *config.Config, vmManager *vm.Manager) ([]txhandler.PostCommitMiddleware, error) { +func postCommitMiddleWAre(cfg config.Config, vmManager vm.Manager) ([]txhandler.PostCommitMiddleware, error) { postCommitMiddlewares := []txhandler.PostCommitMiddleware{ txhandler.LogPostCommitMiddleware, } @@ -1202,11 +1210,7 @@ func postCommitMiddleWAre(cfg *config.Config, vmManager *vm.Manager) ([]txhandle return postCommitMiddlewares, nil } -type contextFactory func(state appstate.State) (contractpb.Context, error) - -type staticContextFactory func(state appstate.State) (contractpb.StaticContext, error) - -func getContractCtx(pluginName string, vmManager *vm.Manager) func(state appstate.State) (contractpb.Context, error) { +func getContractCtx(pluginName string, vmManager vm.Manager) func(state appstate.State) (contractpb.Context, error) { return func(state appstate.State) (contractpb.Context, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) if err != nil { @@ -1216,7 +1220,7 @@ func getContractCtx(pluginName string, vmManager *vm.Manager) func(state appstat } } -func getContractStaticCtx(pluginName string, vmManager *vm.Manager) func(state appstate.State) (contractpb.StaticContext, error) { +func getContractStaticCtx(pluginName string, vmManager vm.Manager) func(state appstate.State) (contractpb.StaticContext, error) { return func(state appstate.State) (contractpb.StaticContext, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) if err != nil { diff --git a/evm/evm.go b/evm/evm.go index 8a914d768d..0a0316d38b 100644 --- a/evm/evm.go +++ b/evm/evm.go @@ -29,6 +29,7 @@ import ( const ( EVMEnabled = true defaultGasLimit = math.MaxUint64 + //defaultTraceTimeout = 5 * time.Second ) //Metrics @@ -148,17 +149,22 @@ type Evm struct { gasLimit uint64 } -func NewEvm(sdb vm.StateDB, lstate state.State, abm *evmAccountBalanceManager, debug bool) *Evm { +func NewEvm( + sdb vm.StateDB, lstate state.State, abm *evmAccountBalanceManager, debug bool, tracer *vm.Tracer, +) (*Evm, error) { + p := new(Evm) p.sdb = sdb p.gasLimit = lstate.Config().GetEvm().GetGasLimit() if p.gasLimit == 0 { p.gasLimit = defaultGasLimit } - p.chainConfig = defaultChainConfig(lstate.FeatureEnabled(features.EvmConstantinopleFeature, false)) - - p.vmConfig = defaultVmConfig(debug) + var err error + p.vmConfig, err = createVmConfig(debug, tracer) + if err != nil { + return nil, errors.Wrap(err, "creating vm.Config") + } p.validateTxValue = lstate.FeatureEnabled(features.CheckTxValueFeature, false) p.context = vm.Context{ CanTransfer: core.CanTransfer, @@ -181,7 +187,7 @@ func NewEvm(sdb vm.StateDB, lstate state.State, abm *evmAccountBalanceManager, d abm.Transfer(from, to, amount) } } - return p + return p, nil } func (e Evm) Create(caller loom.Address, code []byte, value *loom.BigUInt) ([]byte, loom.Address, error) { @@ -300,31 +306,15 @@ func defaultChainConfig(enableConstantinople bool) params.ChainConfig { } } -func defaultVmConfig(evmDebuggingEnabled bool) vm.Config { - logCfg := vm.LogConfig{ - DisableMemory: true, // disable memory capture - DisableStack: true, // disable stack capture - DisableStorage: true, // disable storage capture - Limit: 0, // maximum length of output, but zero means unlimited - } - debug := false - +func createVmConfig(evmDebuggingEnabled bool, tracer *vm.Tracer) (vm.Config, error) { if evmDebuggingEnabled { log.Error("WARNING!!!! EVM Debug mode enabled, do NOT run this on a production server!!!") - logCfg = vm.LogConfig{ - DisableMemory: true, // disable memory capture - DisableStack: true, // disable stack capture - DisableStorage: true, // disable storage capture - Limit: 0, // maximum length of output, but zero means unlimited - } - debug = true } - logger := vm.NewStructLogger(&logCfg) return vm.Config{ // Debug enabled debugging Interpreter options - Debug: debug, + Debug: evmDebuggingEnabled, // Tracer is the op code logger - Tracer: logger, + Tracer: *tracer, // NoRecursion disabled Interpreter call, callcode, // delegate call and create. NoRecursion: false, @@ -334,5 +324,5 @@ func defaultVmConfig(evmDebuggingEnabled bool) vm.Config { // may be left uninitialised and will be set to the default // table. //JumpTable: [256]operation, - } + }, nil } diff --git a/evm/loomevm.go b/evm/loomevm.go index 555f424f93..814af47db0 100644 --- a/evm/loomevm.go +++ b/evm/loomevm.go @@ -55,7 +55,7 @@ type LoomEvm struct { // TODO: this doesn't need to be exported, rename to newLoomEvmWithState func NewLoomEvm( loomState appstate.State, accountBalanceManager AccountBalanceManager, - logContext *ethdbLogContext, debug bool, + logContext *ethdbLogContext, debug bool, tracer *ethvm.Tracer, ) (*LoomEvm, error) { p := new(LoomEvm) p.db = NewLoomEthdb(loomState, logContext) @@ -75,7 +75,10 @@ func NewLoomEvm( return nil, err } - p.Evm = NewEvm(p.sdb, loomState, abm, debug) + p.Evm, err = NewEvm(p.sdb, loomState, abm, debug, tracer) + if err != nil { + return nil, errors.Wrap(err, "creating tracer") + } return p, nil } @@ -112,7 +115,7 @@ var LoomVmFactory = func(state appstate.State) (vm.VM, error) { nil, ) receiptHandler := receiptHandlerProvider.Writer() - return NewLoomVm(state, eventHandler, receiptHandler, nil, debug), nil + return NewLoomVm(state, receiptHandler, nil, debug, nil), nil } // LoomVm implements the loomchain/vm.VM interface using the EVM. @@ -122,20 +125,22 @@ type LoomVm struct { receiptHandler loomchain.WriteReceiptHandler createABM AccountBalanceManagerFactoryFunc debug bool + tracer *ethvm.Tracer } func NewLoomVm( loomState appstate.State, - eventHandler loomchain.EventHandler, receiptHandler loomchain.WriteReceiptHandler, createABM AccountBalanceManagerFactoryFunc, debug bool, + tracer *ethvm.Tracer, ) vm.VM { return &LoomVm{ state: loomState, receiptHandler: receiptHandler, createABM: createABM, debug: debug, + tracer: tracer, } } @@ -152,7 +157,7 @@ func (lvm LoomVm) Create(caller loom.Address, code []byte, value *loom.BigUInt) contractAddr: loom.Address{}, callerAddr: caller, } - levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug) + levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug, lvm.tracer) if err != nil { return nil, loom.Address{}, err } @@ -207,7 +212,7 @@ func (lvm LoomVm) Call(caller, addr loom.Address, input []byte, value *loom.BigU contractAddr: addr, callerAddr: caller, } - levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug) + levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug, lvm.tracer) if err != nil { return nil, err } @@ -247,7 +252,7 @@ func (lvm LoomVm) Call(caller, addr loom.Address, input []byte, value *loom.BigU } func (lvm LoomVm) StaticCall(caller, addr loom.Address, input []byte) ([]byte, error) { - levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(true), nil, lvm.debug) + levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(true), nil, lvm.debug, lvm.tracer) if err != nil { return nil, err } @@ -255,7 +260,7 @@ func (lvm LoomVm) StaticCall(caller, addr loom.Address, input []byte) ([]byte, e } func (lvm LoomVm) GetCode(addr loom.Address) ([]byte, error) { - levm, err := NewLoomEvm(lvm.state, nil, nil, lvm.debug) + levm, err := NewLoomEvm(lvm.state, nil, nil, lvm.debug, lvm.tracer) if err != nil { return nil, err } @@ -263,7 +268,7 @@ func (lvm LoomVm) GetCode(addr loom.Address) ([]byte, error) { } func (lvm LoomVm) GetStorageAt(addr loom.Address, key []byte) ([]byte, error) { - levm, err := NewLoomEvm(lvm.state, nil, nil, lvm.debug) + levm, err := NewLoomEvm(lvm.state, nil, nil, lvm.debug, lvm.tracer) if err != nil { return nil, err } diff --git a/evm/noevm.go b/evm/noevm.go index 911f0fb93d..ec49e6c29b 100644 --- a/evm/noevm.go +++ b/evm/noevm.go @@ -16,11 +16,12 @@ var ( const EVMEnabled = false func NewLoomVm( - loomState state.State, - eventHandler loomchain.EventHandler, - receiptHandler loomchain.WriteReceiptHandler, - createABM AccountBalanceManagerFactoryFunc, - debug bool, + _ state.State, + _ loomchain.EventHandler, + _ loomchain.WriteReceiptHandler, + _ AccountBalanceManagerFactoryFunc, + _ bool, + _ *eth.TraceConfig, ) lvm.VM { return nil } diff --git a/integration_tests/ethcoin_evm_test.go b/integration_tests/ethcoin_evm_test.go index fbbd24b9f2..07557fee96 100644 --- a/integration_tests/ethcoin_evm_test.go +++ b/integration_tests/ethcoin_evm_test.go @@ -208,7 +208,7 @@ func (c *ethCoinIntegrationTestHelper) callEVM( if err != nil { return err } - vm := evm.NewLoomVm(ctx.State, nil, nil, ctx.AccountBalanceManager, false) + vm := evm.NewLoomVm(ctx.State, nil, ctx.AccountBalanceManager, false, nil) _, err = vm.Call(ctx.Message().Sender, c.Address, input, loom.NewBigUIntFromInt(0)) if err != nil { return err @@ -223,7 +223,7 @@ func (c *ethCoinIntegrationTestHelper) staticCallEVM( if err != nil { return err } - vm := evm.NewLoomVm(ctx.State, nil, nil, ctx.AccountBalanceManager, false) + vm := evm.NewLoomVm(ctx.State, nil, ctx.AccountBalanceManager, false, nil) output, err := vm.StaticCall(ctx.Message().Sender, c.Address, input) if err != nil { return err @@ -239,7 +239,7 @@ func deployContractToEVM(ctx *plugin.FakeContextWithEVM, filename string, caller } byteCode := common.FromHex(string(hexByteCode)) - vm := evm.NewLoomVm(ctx.State, nil, nil, nil, false) + vm := evm.NewLoomVm(ctx.State, nil, nil, false, nil) _, contractAddr, err = vm.Create(caller, byteCode, loom.NewBigUIntFromInt(0)) if err != nil { return contractAddr, err diff --git a/plugin/fake_context.go b/plugin/fake_context.go index 4aa075ba0c..c7b497dfb7 100644 --- a/plugin/fake_context.go +++ b/plugin/fake_context.go @@ -103,7 +103,7 @@ func (c *FakeContextWithEVM) CallEVM(addr loom.Address, input []byte, value *loo if c.useAccountBalanceManager { createABM = c.AccountBalanceManager } - vm := levm.NewLoomVm(c.State, nil, nil, createABM, false) + vm := levm.NewLoomVm(c.State, nil, createABM, false, nil) return vm.Call(c.ContractAddress(), addr, input, value) } @@ -112,7 +112,7 @@ func (c *FakeContextWithEVM) StaticCallEVM(addr loom.Address, input []byte) ([]b if c.useAccountBalanceManager { createABM = c.AccountBalanceManager } - vm := levm.NewLoomVm(c.State, nil, nil, createABM, false) + vm := levm.NewLoomVm(c.State, nil, createABM, false, nil) return vm.StaticCall(c.ContractAddress(), addr, input) } diff --git a/plugin/vm.go b/plugin/vm.go index 893e48bd71..a059a01ad0 100644 --- a/plugin/vm.go +++ b/plugin/vm.go @@ -197,7 +197,7 @@ func (vm *PluginVM) CallEVM(caller, addr loom.Address, input []byte, value *loom return nil, err } } - evm := levm.NewLoomVm(vm.State, vm.EventHandler, vm.receiptWriter, createABM, false) + evm := levm.NewLoomVm(vm.State, vm.receiptWriter, createABM, false, nil) return evm.Call(caller, addr, input, value) } @@ -210,7 +210,7 @@ func (vm *PluginVM) StaticCallEVM(caller, addr loom.Address, input []byte) ([]by return nil, err } } - evm := levm.NewLoomVm(vm.State, vm.EventHandler, vm.receiptWriter, createABM, false) + evm := levm.NewLoomVm(vm.State, vm.receiptWriter, createABM, false, nil) return evm.StaticCall(caller, addr, input) } diff --git a/plugin/vm_test.go b/plugin/vm_test.go index 3875f7124c..947dc1c1af 100644 --- a/plugin/vm_test.go +++ b/plugin/vm_test.go @@ -144,7 +144,7 @@ func TestPluginVMContractContextCaller(t *testing.T) { require.NoError(t, err) vm := NewPluginVM(loader, state, createRegistry(state), &fakeEventHandler{}, nil, nil, nil, nil) - evm := levm.NewLoomVm(state, nil, nil, nil, false) + evm := levm.NewLoomVm(state, nil, nil, false, nil) // Deploy contracts owner := loom.RootAddress("chain") @@ -210,7 +210,7 @@ func TestGetEvmTxReceipt(t *testing.T) { require.NoError(t, err) receiptHandler := handler.NewReceiptHandler( loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher()), - common.DefaultMaxReceipts, + rcommon.DefaultMaxReceipts, evmAuxStore, ) require.NoError(t, err) @@ -240,7 +240,7 @@ func TestGetEvmTxReceiptNoCommit(t *testing.T) { require.NoError(t, err) receiptHandler := handler.NewReceiptHandler( loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher()), - common.DefaultMaxReceipts, + rcommon.DefaultMaxReceipts, evmAuxStore, ) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index f107971772..565bf96d23 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -3,11 +3,14 @@ package debug import ( + "fmt" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" - "github.com/pkg/errors" - + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/loomnetwork/loomchain/store" "github.com/loomnetwork/loomchain/txhandler/middleware" + "github.com/pkg/errors" ) func TraceTransaction( @@ -26,10 +29,18 @@ func TraceTransaction( tx := block.Block.Data.Txs[i] _, _ = app.ProcessTx(tx) } + result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) - result, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) - if err != nil { - return nil, err + switch tracer := tracer.(type) { + case *vm.StructLogger: + return ðapi.ExecutionResult{ + Failed: err == nil, + ReturnValue: fmt.Sprintf("%x", result), + StructLogs: ethapi.FormatLogs(tracer.StructLogs()), + }, nil + case *tracers.Tracer: + return tracer.GetResult() + default: + return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) } - return result, errors.New("not implemented") } diff --git a/rpc/query_server.go b/rpc/query_server.go index 514599ff26..8727e53633 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -277,7 +277,7 @@ func (s *QueryServer) queryEvm(caller, contract loom.Address, query []byte) ([]b return nil, err } } - vm := levm.NewLoomVm(snapshot, nil, nil, createABM, false) + vm := levm.NewLoomVm(snapshot, nil, createABM, false, nil) return vm.StaticCall(callerAddr, contract, query) } @@ -318,7 +318,7 @@ func (s *QueryServer) GetEvmCode(contract string) ([]byte, error) { snapshot := s.StateProvider.ReadOnlyState() defer snapshot.Release() - vm := levm.NewLoomVm(snapshot, nil, nil, nil, false) + vm := levm.NewLoomVm(snapshot, nil, nil, false, nil) return vm.GetCode(contractAddr) } @@ -332,7 +332,7 @@ func (s *QueryServer) EthGetCode(address eth.Data, block eth.BlockHeight) (eth.D snapshot := s.StateProvider.ReadOnlyState() defer snapshot.Release() - evm := levm.NewLoomVm(snapshot, nil, nil, nil, false) + evm := levm.NewLoomVm(snapshot, nil, nil, false, nil) code, err := evm.GetCode(addr) if err != nil { return "", errors.Wrapf(err, "getting evm code for %v", address) @@ -1085,7 +1085,7 @@ func (s *QueryServer) EthGetStorageAt(local eth.Data, position string, block eth return "", errors.Wrapf(err, "unable to get storage at height %v", block) } - evm := levm.NewLoomVm(snapshot, nil, nil, nil, false) + evm := levm.NewLoomVm(snapshot, nil, nil, false, nil) storage, err := evm.GetStorageAt(address, ethcommon.HexToHash(position).Bytes()) if err != nil { return "", errors.Wrapf(err, "failed to get EVM storage at %v", address.Local.String()) diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go new file mode 100644 index 0000000000..26490a8e5e --- /dev/null +++ b/txhandler/factory/handler_factory.go @@ -0,0 +1,248 @@ +package factory + +import ( + "github.com/ethereum/go-ethereum/eth" + "github.com/loomnetwork/go-loom/plugin/contractpb" + "github.com/pkg/errors" + + ethvm "github.com/ethereum/go-ethereum/core/vm" + "github.com/gogo/protobuf/proto" + "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/config" + "github.com/loomnetwork/loomchain/evm" + "github.com/loomnetwork/loomchain/migrations" + "github.com/loomnetwork/loomchain/plugin" + "github.com/loomnetwork/loomchain/registry/factory" + "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/throttle" + "github.com/loomnetwork/loomchain/tx_handler" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/txhandler/middleware" + "github.com/loomnetwork/loomchain/vm" +) + +func NewTxHandlerFactory( + cfg config.Config, + vmManager *vm.Manager, + chainID string, + store store.VersionedKVStore, +) txhandler.TxHandlerFactory { + return txHandleFactory{ + cfg: cfg, + vmManager: vmManager, + chainID: chainID, + store: store, + } +} + +type txHandleFactory struct { + cfg config.Config + vmManager *vm.Manager + chainID string + store store.VersionedKVStore + createRegistry factory.RegistryFactoryFunc +} + +func (f txHandleFactory) TxHandler(tracer *ethvm.Tracer) (txhandler.TxHandler, error) { + vmManager := createVmManager(f.vmManager, tracer) + + txMiddleware, err := txMiddleWare(f.cfg, vmManager, f.chainID, f.store) + if err != nil { + return nil, err + } + postCommitMiddlewares, err := postCommitMiddleWAre(f.cfg, vmManager) + if err != nil { + return nil, err + } + + return txhandler.MiddlewareTxHandler( + txMiddleware, + router(f.cfg, vmManager, f.createRegistry), + postCommitMiddlewares, + ), nil +} + +func createVmManager(vmManager *vm.Manager, tracer *ethvm.Tracer) vm.Manager { + if tracer == nil && vmManager != nil { + return *vmManager + } + managerWithTracer := vm.NewManager() + managerWithTracer.Register(vm.VMType_EVM, func(_state state.State) (vm.VM, error) { + var createABM evm.AccountBalanceManagerFactoryFunc + return evm.NewLoomVm(_state, nil, createABM, false, tracer), nil + }) + return *managerWithTracer +} + +func txMiddleWare( + cfg config.Config, + vmManager vm.Manager, + chainID string, + appStore store.VersionedKVStore, +) ([]txhandler.TxMiddleware, error) { + txMiddleWare := []txhandler.TxMiddleware{ + txhandler.LogTxMiddleware, + txhandler.RecoveryTxMiddleware, + } + + txMiddleWare = append(txMiddleWare, auth.NewChainConfigMiddleware( + cfg.Auth, + getContractStaticCtx("addressmapper", vmManager), + )) + + if cfg.Karma.Enabled { + txMiddleWare = append(txMiddleWare, throttle.GetKarmaMiddleWare( + cfg.Karma.Enabled, + cfg.Karma.MaxCallCount, + cfg.Karma.SessionDuration, + getContractCtx("karma", vmManager), + )) + } + + if cfg.TxLimiter.Enabled { + txMiddleWare = append(txMiddleWare, middleware.NewTxLimiterMiddleware(cfg.TxLimiter)) + } + + if cfg.ContractTxLimiter.Enabled { + udwCtxFactory := getContractCtx("user-deployer-whitelist", vmManager) + txMiddleWare = append( + txMiddleWare, middleware.NewContractTxLimiterMiddleware(cfg.ContractTxLimiter, udwCtxFactory), + ) + } + + if cfg.DeployerWhitelist.ContractEnabled { + dwMiddleware, err := middleware.NewDeployerWhitelistMiddleware(getContractCtx("deployerwhitelist", vmManager)) + if err != nil { + return nil, err + } + txMiddleWare = append(txMiddleWare, dwMiddleware) + + } + + txMiddleWare = append(txMiddleWare, auth.NonceTxMiddleware(appStore)) + + if cfg.GoContractDeployerWhitelist.Enabled { + goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) + if err != nil { + return nil, errors.Wrapf(err, "getting list of users allowed go deploys") + } + txMiddleWare = append(txMiddleWare, throttle.GetGoDeployTxMiddleWare(goDeployers)) + } + + txMiddleWare = append(txMiddleWare, txhandler.NewInstrumentingTxMiddleware()) + + return txMiddleWare, nil +} + +func router( + cfg config.Config, + vmManager vm.Manager, + createRegistry factory.RegistryFactoryFunc, +) txhandler.TxHandler { + router := middleware.NewTxRouter() + isEvmTx := func(txID uint32, _ state.State, txBytes []byte, isCheckTx bool) bool { + var msg vm.MessageTx + err := proto.Unmarshal(txBytes, &msg) + if err != nil { + return false + } + + switch txID { + case 1: + var tx vm.DeployTx + err = proto.Unmarshal(msg.Data, &tx) + if err != nil { + // In case of error, let's give safest response, + // let's TxHandler down the line, handle it. + return false + } + return tx.VmType == vm.VMType_EVM + case 2: + var tx vm.CallTx + err = proto.Unmarshal(msg.Data, &tx) + if err != nil { + // In case of error, let's give safest response, + // let's TxHandler down the line, handle it. + return false + } + return tx.VmType == vm.VMType_EVM + case 3: + return false + default: + return false + } + } + + deployTxHandler := &vm.DeployTxHandler{ + Manager: &vmManager, + CreateRegistry: createRegistry, + AllowNamedEVMContracts: cfg.AllowNamedEvmContracts, + } + + callTxHandler := &vm.CallTxHandler{ + Manager: &vmManager, + } + + migrationTxHandler := &tx_handler.MigrationTxHandler{ + Manager: &vmManager, + CreateRegistry: createRegistry, + Migrations: map[int32]tx_handler.MigrationFunc{ + 1: migrations.DPOSv3Migration, + 2: migrations.GatewayMigration, + 3: migrations.GatewayMigration, + }, + } + + router.HandleDeliverTx(1, middleware.GeneratePassthroughRouteHandler(deployTxHandler)) + router.HandleDeliverTx(2, middleware.GeneratePassthroughRouteHandler(callTxHandler)) + router.HandleDeliverTx(3, middleware.GeneratePassthroughRouteHandler(migrationTxHandler)) + + // TODO: Write this in more elegant way + router.HandleCheckTx(1, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, deployTxHandler)) + router.HandleCheckTx(2, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, callTxHandler)) + router.HandleCheckTx(3, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, migrationTxHandler)) + + return router +} + +func postCommitMiddleWAre(cfg config.Config, vmManager vm.Manager) ([]txhandler.PostCommitMiddleware, error) { + postCommitMiddlewares := []txhandler.PostCommitMiddleware{ + txhandler.LogPostCommitMiddleware, + } + + if cfg.UserDeployerWhitelist.ContractEnabled { + udwContractFactory := getContractCtx("user-deployer-whitelist", vmManager) + evmDeployRecorderMiddleware, err := middleware.NewEVMDeployRecorderPostCommitMiddleware(udwContractFactory) + if err != nil { + return nil, err + } + postCommitMiddlewares = append(postCommitMiddlewares, evmDeployRecorderMiddleware) + } + + // We need to make sure nonce post commit middleware is last as + // it doesn't pass control to other middlewares after it. + postCommitMiddlewares = append(postCommitMiddlewares, auth.NonceTxPostNonceMiddleware) + + return postCommitMiddlewares, nil +} + +func getContractCtx(pluginName string, vmManager vm.Manager) func(_ state.State) (contractpb.Context, error) { + return func(_state state.State) (contractpb.Context, error) { + pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, _state) + if err != nil { + return nil, err + } + return plugin.NewInternalContractContext(pluginName, pvm.(*plugin.PluginVM), false) + } +} + +func getContractStaticCtx(pluginName string, vmManager vm.Manager) func(_ state.State) (contractpb.StaticContext, error) { + return func(_state state.State) (contractpb.StaticContext, error) { + pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, _state) + if err != nil { + return nil, err + } + return plugin.NewInternalContractContext(pluginName, pvm.(*plugin.PluginVM), true) + } +} diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go index cdf9a151f1..adccdc4dae 100644 --- a/txhandler/middleware/memory_app.go +++ b/txhandler/middleware/memory_app.go @@ -3,7 +3,9 @@ package middleware import ( "context" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/loomnetwork/go-loom/builtin/types/chainconfig" abci "github.com/tendermint/tendermint/abci/types" @@ -14,18 +16,19 @@ import ( type InMemoryApp interface { ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) - TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, error) + TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) } type inMemoryApp struct { - LastBlockHeader abci.Header - CurBlockHeader abci.Header - CurBlockHash []byte - Store store.VersionedKVStore - TxHandler txhandler.TxHandler - ReceiptsVersion int32 - GetValidatorSet state.GetValidatorSet - config *chainconfig.Config + lastBlockHeader abci.Header + curBlockHeader abci.Header + curBlockHash []byte + store store.VersionedKVStore + txHandler txhandler.TxHandler + receiptsVersion int32 + getValidatorSet state.GetValidatorSet + config *chainconfig.Config + txHandlerFactory txhandler.TxHandlerFactory } func NewInMemoryApp( @@ -37,44 +40,59 @@ func NewInMemoryApp( receiptsVersion int32, getValidatorSet state.GetValidatorSet, config *chainconfig.Config, + txHandlerFactory txhandler.TxHandlerFactory, ) InMemoryApp { return &inMemoryApp{ - LastBlockHeader: lastBlockHeader, - CurBlockHeader: curBlockHeader, - CurBlockHash: curBlockHash, - Store: store, - TxHandler: txHandler, - ReceiptsVersion: receiptsVersion, - GetValidatorSet: getValidatorSet, - config: config, + lastBlockHeader: lastBlockHeader, + curBlockHeader: curBlockHeader, + curBlockHash: curBlockHash, + store: store, + txHandler: txHandler, + receiptsVersion: receiptsVersion, + getValidatorSet: getValidatorSet, + config: config, + txHandlerFactory: txHandlerFactory, } } func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) { - return ma.processTx(txBytes, ma.TxHandler) + return ma.processTx(txBytes, ma.txHandler) } func (ma *inMemoryApp) processTx(txBytes []byte, txHandler txhandler.TxHandler) (txhandler.TxHandlerResult, error) { - splitStoreTx := store.WrapAtomic(ma.Store).BeginTx() + splitStoreTx := store.WrapAtomic(ma.store).BeginTx() defer splitStoreTx.Rollback() storeState := state.NewStoreState( context.Background(), splitStoreTx, - ma.CurBlockHeader, - ma.CurBlockHash, - ma.GetValidatorSet, + ma.curBlockHeader, + ma.curBlockHash, + ma.getValidatorSet, ).WithOnChainConfig(ma.config) return txHandler.ProcessTx(storeState, txBytes, false) } -func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, error) { - traceTxHandle, err := TraceTxHandle(traceCfg) +func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) { + tracer, err := createTracer(traceCfg) if err != nil { - return txhandler.TxHandlerResult{}, err + return txhandler.TxHandlerResult{}, nil, err } - return ma.processTx(txBytes, traceTxHandle) + + traceTxHandle, err := ma.txHandlerFactory.TxHandler(&tracer) + if err != nil { + return txhandler.TxHandlerResult{}, nil, err + } + res, err := ma.processTx(txBytes, traceTxHandle) + return res, tracer, err } -func TraceTxHandle(traceCfg eth.TraceConfig) (txhandler.TxHandler, error) { - return nil, nil +func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { + if traceCfg.Tracer == nil { + return vm.NewStructLogger(traceCfg.LogConfig), nil + } + tracer, err := tracers.New(*traceCfg.Tracer) + if err != nil { + return nil, err + } + return tracer, nil } diff --git a/txhandler/txhandler.go b/txhandler/txhandler.go index 1af2bd32b1..cacbfbd2e2 100644 --- a/txhandler/txhandler.go +++ b/txhandler/txhandler.go @@ -1,6 +1,7 @@ package txhandler import ( + "github.com/ethereum/go-ethereum/core/vm" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/common" @@ -25,3 +26,7 @@ type TxHandlerResult struct { func (f TxHandlerFunc) ProcessTx(s state.State, txBytes []byte, isCheckTx bool) (TxHandlerResult, error) { return f(s, txBytes, isCheckTx) } + +type TxHandlerFactory interface { + TxHandler(tracer *vm.Tracer) (TxHandler, error) +} From 2b94ec352b40b35f92ebc001758028b7a95ee3de Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 24 Sep 2019 16:25:39 +0100 Subject: [PATCH 03/49] wip implement --- cmd/loom/loom.go | 2 +- evm/evm.go | 16 ++++++++++++---- evm/loomevm.go | 6 +++--- rpc/debug/trace.go | 6 ++++-- txhandler/factory/handler_factory.go | 16 +++++++++------- txhandler/middleware/memory_app.go | 4 ++-- txhandler/txhandler.go | 2 +- 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index 1ac3c021aa..ca4f20ddf4 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -985,7 +985,7 @@ func loadApp( //if err != nil { // return nil, err //} - txHandlerFactory := factory.NewTxHandlerFactory(*cfg, vmManager, chainID, appStore) + txHandlerFactory := factory.NewTxHandlerFactory(*cfg, vmManager, chainID, appStore, createRegistry) chainTxHandler, err := txHandlerFactory.TxHandler(nil) if err != nil { return nil, err diff --git a/evm/evm.go b/evm/evm.go index 0a0316d38b..ed24a7110d 100644 --- a/evm/evm.go +++ b/evm/evm.go @@ -150,9 +150,8 @@ type Evm struct { } func NewEvm( - sdb vm.StateDB, lstate state.State, abm *evmAccountBalanceManager, debug bool, tracer *vm.Tracer, + sdb vm.StateDB, lstate state.State, abm *evmAccountBalanceManager, debug bool, tracer vm.Tracer, ) (*Evm, error) { - p := new(Evm) p.sdb = sdb p.gasLimit = lstate.Config().GetEvm().GetGasLimit() @@ -306,15 +305,24 @@ func defaultChainConfig(enableConstantinople bool) params.ChainConfig { } } -func createVmConfig(evmDebuggingEnabled bool, tracer *vm.Tracer) (vm.Config, error) { +func createVmConfig(evmDebuggingEnabled bool, tracer vm.Tracer) (vm.Config, error) { if evmDebuggingEnabled { log.Error("WARNING!!!! EVM Debug mode enabled, do NOT run this on a production server!!!") } + if tracer == nil { + logCfg := vm.LogConfig{ + DisableMemory: true, // disable memory capture + DisableStack: true, // disable stack capture + DisableStorage: true, // disable storage capture + Limit: 0, // maximum length of output, but zero means unlimited + } + tracer = vm.NewStructLogger(&logCfg) + } return vm.Config{ // Debug enabled debugging Interpreter options Debug: evmDebuggingEnabled, // Tracer is the op code logger - Tracer: *tracer, + Tracer: tracer, // NoRecursion disabled Interpreter call, callcode, // delegate call and create. NoRecursion: false, diff --git a/evm/loomevm.go b/evm/loomevm.go index 814af47db0..e50a2ac58e 100644 --- a/evm/loomevm.go +++ b/evm/loomevm.go @@ -55,7 +55,7 @@ type LoomEvm struct { // TODO: this doesn't need to be exported, rename to newLoomEvmWithState func NewLoomEvm( loomState appstate.State, accountBalanceManager AccountBalanceManager, - logContext *ethdbLogContext, debug bool, tracer *ethvm.Tracer, + logContext *ethdbLogContext, debug bool, tracer ethvm.Tracer, ) (*LoomEvm, error) { p := new(LoomEvm) p.db = NewLoomEthdb(loomState, logContext) @@ -125,7 +125,7 @@ type LoomVm struct { receiptHandler loomchain.WriteReceiptHandler createABM AccountBalanceManagerFactoryFunc debug bool - tracer *ethvm.Tracer + tracer ethvm.Tracer } func NewLoomVm( @@ -133,7 +133,7 @@ func NewLoomVm( receiptHandler loomchain.WriteReceiptHandler, createABM AccountBalanceManagerFactoryFunc, debug bool, - tracer *ethvm.Tracer, + tracer ethvm.Tracer, ) vm.VM { return &LoomVm{ state: loomState, diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 565bf96d23..1350c5a008 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -4,13 +4,15 @@ package debug import ( "fmt" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethapi" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/pkg/errors" + "github.com/loomnetwork/loomchain/store" "github.com/loomnetwork/loomchain/txhandler/middleware" - "github.com/pkg/errors" ) func TraceTransaction( diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 26490a8e5e..4198b699a0 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -1,12 +1,12 @@ package factory import ( - "github.com/ethereum/go-ethereum/eth" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/pkg/errors" ethvm "github.com/ethereum/go-ethereum/core/vm" "github.com/gogo/protobuf/proto" + "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/config" "github.com/loomnetwork/loomchain/evm" @@ -27,12 +27,14 @@ func NewTxHandlerFactory( vmManager *vm.Manager, chainID string, store store.VersionedKVStore, + createRegistry factory.RegistryFactoryFunc, ) txhandler.TxHandlerFactory { return txHandleFactory{ - cfg: cfg, - vmManager: vmManager, - chainID: chainID, - store: store, + cfg: cfg, + vmManager: vmManager, + chainID: chainID, + store: store, + createRegistry: createRegistry, } } @@ -44,7 +46,7 @@ type txHandleFactory struct { createRegistry factory.RegistryFactoryFunc } -func (f txHandleFactory) TxHandler(tracer *ethvm.Tracer) (txhandler.TxHandler, error) { +func (f txHandleFactory) TxHandler(tracer ethvm.Tracer) (txhandler.TxHandler, error) { vmManager := createVmManager(f.vmManager, tracer) txMiddleware, err := txMiddleWare(f.cfg, vmManager, f.chainID, f.store) @@ -63,7 +65,7 @@ func (f txHandleFactory) TxHandler(tracer *ethvm.Tracer) (txhandler.TxHandler, e ), nil } -func createVmManager(vmManager *vm.Manager, tracer *ethvm.Tracer) vm.Manager { +func createVmManager(vmManager *vm.Manager, tracer ethvm.Tracer) vm.Manager { if tracer == nil && vmManager != nil { return *vmManager } diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go index adccdc4dae..7648a1e32f 100644 --- a/txhandler/middleware/memory_app.go +++ b/txhandler/middleware/memory_app.go @@ -77,8 +77,8 @@ func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) if err != nil { return txhandler.TxHandlerResult{}, nil, err } - - traceTxHandle, err := ma.txHandlerFactory.TxHandler(&tracer) + // Pass pointer to tracer as we want evm to run with this object, not a copy + traceTxHandle, err := ma.txHandlerFactory.TxHandler(tracer) if err != nil { return txhandler.TxHandlerResult{}, nil, err } diff --git a/txhandler/txhandler.go b/txhandler/txhandler.go index cacbfbd2e2..a2b91b137f 100644 --- a/txhandler/txhandler.go +++ b/txhandler/txhandler.go @@ -28,5 +28,5 @@ func (f TxHandlerFunc) ProcessTx(s state.State, txBytes []byte, isCheckTx bool) } type TxHandlerFactory interface { - TxHandler(tracer *vm.Tracer) (TxHandler, error) + TxHandler(tracer vm.Tracer) (TxHandler, error) } From 00743d86aa18f84deddf69d1e3925865bd289d3c Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 24 Sep 2019 18:58:08 +0100 Subject: [PATCH 04/49] implement --- cmd/loom/loom.go | 2 +- rpc/debug/jsonrpc_conversion.go | 40 +++++++++++++++++----------- rpc/instrumenting.go | 2 +- rpc/mock_query_server.go | 2 +- rpc/query_server.go | 6 ++--- rpc/query_service.go | 2 +- txhandler/factory/handler_factory.go | 9 ++++--- txhandler/middleware/memory_app.go | 5 ++-- txhandler/txhandler.go | 2 +- 9 files changed, 41 insertions(+), 29 deletions(-) diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index ca4f20ddf4..c56f4f5593 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -986,7 +986,7 @@ func loadApp( // return nil, err //} txHandlerFactory := factory.NewTxHandlerFactory(*cfg, vmManager, chainID, appStore, createRegistry) - chainTxHandler, err := txHandlerFactory.TxHandler(nil) + chainTxHandler, err := txHandlerFactory.TxHandler(nil, true) if err != nil { return nil, err } diff --git a/rpc/debug/jsonrpc_conversion.go b/rpc/debug/jsonrpc_conversion.go index c920cbb9fc..ad92ed00b7 100644 --- a/rpc/debug/jsonrpc_conversion.go +++ b/rpc/debug/jsonrpc_conversion.go @@ -4,26 +4,34 @@ package debug import ( "github.com/ethereum/go-ethereum/core/vm" - etheth "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth" ) type JsonTraceConfig struct { - DisableStorage bool `json:"disableStorage,omitempty"` - DisableMemory bool `json:"disableMemory,omitempty"` - DisableStack bool `json:"disableStack,omitempty"` - Tracer string `json:"tracer,omitempty"` - Timeout string `json:"address,omitempty"` + LogConfig *JsonLogConfig `json:"logconfig,omitempty"` + Tracer string `json:"tracer,omitempty"` + Timeout string `json:"address,omitempty"` } -func DecTraceConfig(jcfg JsonTraceConfig) etheth.TraceConfig { - return etheth.TraceConfig{ - LogConfig: &vm.LogConfig{ - DisableMemory: jcfg.DisableMemory, - DisableStack: jcfg.DisableStack, - DisableStorage: jcfg.DisableStorage, - }, - Tracer: &jcfg.Tracer, - Timeout: &jcfg.Timeout, - Reexec: nil, +type JsonLogConfig struct { + DisableStorage bool `json:"disableStorage,omitempty"` + DisableMemory bool `json:"disableMemory,omitempty"` + DisableStack bool `json:"disableStack,omitempty"` +} + +func DecTraceConfig(jcfg JsonTraceConfig) eth.TraceConfig { + var logConfig *vm.LogConfig + if jcfg.LogConfig != nil { + logConfig = &vm.LogConfig{ + DisableMemory: jcfg.LogConfig.DisableMemory, + DisableStack: jcfg.LogConfig.DisableStack, + DisableStorage: jcfg.LogConfig.DisableStorage, + } + } + return eth.TraceConfig{ + LogConfig: logConfig, + Tracer: &jcfg.Tracer, + Timeout: &jcfg.Timeout, + Reexec: nil, } } diff --git a/rpc/instrumenting.go b/rpc/instrumenting.go index e464e2f2ce..1d6c320347 100755 --- a/rpc/instrumenting.go +++ b/rpc/instrumenting.go @@ -572,7 +572,7 @@ func (m InstrumentingMiddleware) EthGetTransactionCount( } func (m InstrumentingMiddleware) DebugTraceTransaction( - hash eth.Data, config debug.JsonTraceConfig, + hash eth.Data, config *debug.JsonTraceConfig, ) (resp interface{}, err error) { defer func(begin time.Time) { lvs := []string{"method", "DebugTraceTransaction", "error", fmt.Sprint(err != nil)} diff --git a/rpc/mock_query_server.go b/rpc/mock_query_server.go index d0058b54a4..796eb5fc15 100644 --- a/rpc/mock_query_server.go +++ b/rpc/mock_query_server.go @@ -370,7 +370,7 @@ func (m *MockQueryService) EvmUnSubscribe(id string) (bool, error) { return true, nil } -func (m *MockQueryService) DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) { +func (m *MockQueryService) DebugTraceTransaction(hash eth.Data, config *debug.JsonTraceConfig) (interface{}, error) { m.mutex.Lock() defer m.mutex.Unlock() m.MethodsCalled = append([]string{"DebugTraceTransaction"}, m.MethodsCalled...) diff --git a/rpc/query_server.go b/rpc/query_server.go index 8727e53633..0043212aab 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -1112,9 +1112,9 @@ func (s *QueryServer) EthAccounts() ([]eth.Data, error) { return []eth.Data{}, nil } -func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) { +func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTraceConfig) (interface{}, error) { receipt, err := s.EthGetTransactionReceipt(hash) - if err != nil { + if err != nil || receipt == nil { return nil, errors.Wrap(err, "cant find transaction matching hash") } blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) @@ -1125,7 +1125,7 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config debug.JsonTrac if err != nil { return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) } - cfg := debug.DecTraceConfig(config) + cfg := debug.DecTraceConfig(*config) return debug.TraceTransaction(s.InMemoryApp(blockNumber), s.BlockStore, int64(blockNumber), txIndex, cfg) } diff --git a/rpc/query_service.go b/rpc/query_service.go index d95bd06735..25f24d3946 100755 --- a/rpc/query_service.go +++ b/rpc/query_service.go @@ -67,7 +67,7 @@ type QueryService interface { GetContractRecord(contractAddr string) (*types.ContractRecordResponse, error) // debug transactions. - DebugTraceTransaction(hash eth.Data, config debug.JsonTraceConfig) (interface{}, error) + DebugTraceTransaction(hash eth.Data, config *debug.JsonTraceConfig) (interface{}, error) // deprecated function EvmTxReceipt(txHash []byte) ([]byte, error) diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 4198b699a0..88bfaf575a 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -46,10 +46,10 @@ type txHandleFactory struct { createRegistry factory.RegistryFactoryFunc } -func (f txHandleFactory) TxHandler(tracer ethvm.Tracer) (txhandler.TxHandler, error) { +func (f txHandleFactory) TxHandler(tracer ethvm.Tracer, metrics bool) (txhandler.TxHandler, error) { vmManager := createVmManager(f.vmManager, tracer) - txMiddleware, err := txMiddleWare(f.cfg, vmManager, f.chainID, f.store) + txMiddleware, err := txMiddleWare(f.cfg, vmManager, f.chainID, f.store, metrics) if err != nil { return nil, err } @@ -82,6 +82,7 @@ func txMiddleWare( vmManager vm.Manager, chainID string, appStore store.VersionedKVStore, + metrics bool, ) ([]txhandler.TxMiddleware, error) { txMiddleWare := []txhandler.TxMiddleware{ txhandler.LogTxMiddleware, @@ -132,7 +133,9 @@ func txMiddleWare( txMiddleWare = append(txMiddleWare, throttle.GetGoDeployTxMiddleWare(goDeployers)) } - txMiddleWare = append(txMiddleWare, txhandler.NewInstrumentingTxMiddleware()) + if metrics { + txMiddleWare = append(txMiddleWare, txhandler.NewInstrumentingTxMiddleware()) + } return txMiddleWare, nil } diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go index 7648a1e32f..e189cca317 100644 --- a/txhandler/middleware/memory_app.go +++ b/txhandler/middleware/memory_app.go @@ -77,8 +77,9 @@ func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) if err != nil { return txhandler.TxHandlerResult{}, nil, err } + // Pass pointer to tracer as we want evm to run with this object, not a copy - traceTxHandle, err := ma.txHandlerFactory.TxHandler(tracer) + traceTxHandle, err := ma.txHandlerFactory.TxHandler(tracer, false) if err != nil { return txhandler.TxHandlerResult{}, nil, err } @@ -87,7 +88,7 @@ func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) } func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { - if traceCfg.Tracer == nil { + if traceCfg.Tracer == nil || len(*traceCfg.Tracer) == 0 { return vm.NewStructLogger(traceCfg.LogConfig), nil } tracer, err := tracers.New(*traceCfg.Tracer) diff --git a/txhandler/txhandler.go b/txhandler/txhandler.go index a2b91b137f..28e85d30a2 100644 --- a/txhandler/txhandler.go +++ b/txhandler/txhandler.go @@ -28,5 +28,5 @@ func (f TxHandlerFunc) ProcessTx(s state.State, txBytes []byte, isCheckTx bool) } type TxHandlerFactory interface { - TxHandler(tracer vm.Tracer) (TxHandler, error) + TxHandler(tracer vm.Tracer, metrics bool) (TxHandler, error) } From 239fb4b516534ff18629036c342c2efeaa5ab7ad Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 25 Sep 2019 09:14:13 +0100 Subject: [PATCH 05/49] use ethapi go-ethereum branch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22be3fe768..4162abf16e 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. TG_GIT_REV = HEAD # loomnetwork/go-ethereum loomchain branch -ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9 +ETHEREUM_GIT_REV = ethapi # use go-plugin we get 'timeout waiting for connection info' error HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961 LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9 From 04dd303ad9675d893d05e1372993aae1b0bc5db7 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 25 Sep 2019 10:32:55 +0100 Subject: [PATCH 06/49] use config transfer gateway branch --- Makefile | 2 +- cmd/loom/gateway_reactors.go | 10 ++++++---- config/gateway_config.go | 10 +++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 4162abf16e..f2bc03af29 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ BINANCE_TGORACLE_DIR=$(GOPATH)/src/$(PKG_BINANCE_TGORACLE) # specific commit. GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. -TG_GIT_REV = HEAD +TG_GIT_REV = config # loomnetwork/go-ethereum loomchain branch ETHEREUM_GIT_REV = ethapi # use go-plugin we get 'timeout waiting for connection info' error diff --git a/cmd/loom/gateway_reactors.go b/cmd/loom/gateway_reactors.go index a8b82f90af..e5be654f67 100644 --- a/cmd/loom/gateway_reactors.go +++ b/cmd/loom/gateway_reactors.go @@ -8,10 +8,12 @@ import ( glAuth "github.com/loomnetwork/go-loom/auth" "github.com/loomnetwork/go-loom/client" + tgateway "github.com/loomnetwork/transfer-gateway/gateway" + tconfig "github.com/loomnetwork/transfer-gateway/gateway/config" + "github.com/loomnetwork/loomchain/config" "github.com/loomnetwork/loomchain/fnConsensus" "github.com/loomnetwork/loomchain/log" - tgateway "github.com/loomnetwork/transfer-gateway/gateway" ) const ( @@ -62,7 +64,7 @@ func checkQueryService(name string, chainID string, DAppChainReadURI string, DAp func startGatewayFn( chainID string, fnRegistry fnConsensus.FnRegistry, - cfg *tgateway.TransferGatewayConfig, + cfg *tconfig.TransferGatewayConfig, nodeSigner glAuth.Signer, ) error { if !cfg.BatchSignFnConfig.Enabled { @@ -85,7 +87,7 @@ func startGatewayFn( func startLoomCoinGatewayFn( chainID string, fnRegistry fnConsensus.FnRegistry, - cfg *tgateway.TransferGatewayConfig, + cfg *tconfig.TransferGatewayConfig, nodeSigner glAuth.Signer, ) error { if !cfg.BatchSignFnConfig.Enabled { @@ -107,7 +109,7 @@ func startLoomCoinGatewayFn( func startTronGatewayFn(chainID string, fnRegistry fnConsensus.FnRegistry, - cfg *tgateway.TransferGatewayConfig, + cfg *tconfig.TransferGatewayConfig, nodeSigner glAuth.Signer, ) error { if !cfg.BatchSignFnConfig.Enabled { diff --git a/config/gateway_config.go b/config/gateway_config.go index 2a6649d7c4..d7f1035498 100644 --- a/config/gateway_config.go +++ b/config/gateway_config.go @@ -3,23 +3,23 @@ package config import ( - "github.com/loomnetwork/transfer-gateway/gateway" + "github.com/loomnetwork/transfer-gateway/gateway/config" dpos2cfg "github.com/loomnetwork/transfer-gateway/oracles/dpos2/config" ) -type TransferGatewayConfig = gateway.TransferGatewayConfig +type TransferGatewayConfig = config.TransferGatewayConfig type OracleSerializableConfig = dpos2cfg.OracleSerializableConfig func DefaultTGConfig(rpcProxyPort int32) *TransferGatewayConfig { - return gateway.DefaultConfig(rpcProxyPort) + return config.DefaultConfig(rpcProxyPort) } func DefaultLoomCoinTGConfig(rpcProxyPort int32) *TransferGatewayConfig { - return gateway.DefaultLoomCoinTGConfig(rpcProxyPort) + return config.DefaultLoomCoinTGConfig(rpcProxyPort) } func DefaultTronTGConfig(rpcProxyPort int32) *TransferGatewayConfig { - return gateway.DefaultTronConfig(rpcProxyPort) + return config.DefaultTronConfig(rpcProxyPort) } func DefaultBinanceTGConfig() *TransferGatewayConfig { From 79536ea77dc891d520ea4df00742ea2408cb8ce3 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 25 Sep 2019 10:50:34 +0100 Subject: [PATCH 07/49] touch --- txhandler/middleware/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txhandler/middleware/router.go b/txhandler/middleware/router.go index 8ca1008b5c..3b758b0f41 100644 --- a/txhandler/middleware/router.go +++ b/txhandler/middleware/router.go @@ -14,7 +14,7 @@ type TxRouter struct { checkTxRoutes map[uint32]RouteHandler } -type RouteHandler func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) +type RouteHandler func(txID uint32, _ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) type RouteConditionFunc func(txID uint32, state appstate.State, txBytes []byte, isCheckTx bool) bool From 6002221d45c85ca8def0c27d9a730ae6278c4a7a Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 25 Sep 2019 12:32:38 +0100 Subject: [PATCH 08/49] touch --- cmd/loom/loom.go | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index c56f4f5593..e13081f470 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -965,26 +965,7 @@ func loadApp( return nil, err } } - /* - txMiddleWare, err := middleware.TxMiddleWare(cfg, vmManager, chainID, appStore) - if err != nil { - return nil, err - } - */ - //postCommitMiddlewares, err := postCommitMiddleWAre(*cfg, *vmManager) - //if err != nil { - // return nil, err - //} - - //txHandler, err := middleware.AppTxHandler(cfg, vmManager, createRegistry, chainID, appStore) - //if err != nil { - // return nil, err - //} - - //txMiddleware, err := txMiddleWare(*cfg, *vmManager, chainID, appStore) - //if err != nil { - // return nil, err - //} + txHandlerFactory := factory.NewTxHandlerFactory(*cfg, vmManager, chainID, appStore, createRegistry) chainTxHandler, err := txHandlerFactory.TxHandler(nil, true) if err != nil { @@ -992,13 +973,8 @@ func loadApp( } return &loomchain.Application{ - Store: appStore, - Init: init, - //TxHandler: txhandler.MiddlewareTxHandler( - // txMiddleware, - // router(*cfg, *vmManager, createRegistry), - // postCommitMiddlewares, - //), + Store: appStore, + Init: init, TxHandler: chainTxHandler, TxHandlerFactory: txHandlerFactory, BlockIndexStore: blockIndexStore, From c7af3a81833feebc461037b0690f1c69937f3496 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 25 Sep 2019 16:24:48 +0100 Subject: [PATCH 09/49] add version reader --- app.go | 23 ++++++++---- rpc/debug/trace.go | 48 ++++++++++++++----------- rpc/query_server.go | 8 +++-- state/state.go | 15 ++++++++ store/iavlstore.go | 13 +++++++ store/logstore.go | 8 +++++ store/memstore.go | 8 +++++ store/multi_writer_app_store.go | 8 +++++ store/pruning_iavlstore.go | 8 +++++ store/splitstore.go | 24 ++++++------- store/store.go | 2 ++ store/version_reader.go | 44 +++++++++++++++++++++++ store/versioned_cachingstore.go | 8 +++++ store/versioned_cachingstore_test.go | 8 +++++ txhandler/middleware/memory_app.go | 53 +++++++++++++++++++++------- 15 files changed, 224 insertions(+), 54 deletions(-) create mode 100644 store/version_reader.go diff --git a/app.go b/app.go index b05a20a230..a07e448a50 100644 --- a/app.go +++ b/app.go @@ -9,6 +9,7 @@ import ( "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" + "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" abci "github.com/tendermint/tendermint/abci/types" ttypes "github.com/tendermint/tendermint/types" @@ -606,16 +607,26 @@ func (a *Application) ReadOnlyState() appstate.State { ) } -func (a *Application) InMemoryApp(blockNumber uint64) middleware.InMemoryApp { +func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStore) (middleware.InMemoryApp, error) { + startVersion := blockNumber + for ; a.Store.VersionExists(int64(startVersion)) || startVersion == 0; startVersion-- { + } + if startVersion == 0 { + return nil, errors.Errorf("no saved version for height %d", blockNumber) + } + startStore, err := a.Store.RetrieveVersion(int64(startVersion)) + if err != nil { + return nil, err + } + return middleware.NewInMemoryApp( - a.lastBlockHeader, - a.curBlockHeader, - a.curBlockHash, - a.Store, + int64(startVersion), + blockstore, + startStore, a.TxHandler, a.ReceiptsVersion, a.GetValidatorSet, a.config, a.TxHandlerFactory, - ) + ), nil } diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 1350c5a008..d0c43c59b1 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -22,27 +22,33 @@ func TraceTransaction( txIndex uint64, config eth.TraceConfig, ) (interface{}, error) { - block, err := blockstore.GetBlockByHeight(&blockNumber) - if err != nil { - return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) - } - - for i := uint64(0); i < txIndex; i++ { - tx := block.Block.Data.Txs[i] - _, _ = app.ProcessTx(tx) - } - result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) + for height := app.Height(); height <= blockNumber; height++ { + block, err := blockstore.GetBlockByHeight(&blockNumber) + if err != nil { + return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) + } + for i := uint64(0); i < txIndex; i++ { + if height != blockNumber || i != txIndex { + tx := block.Block.Data.Txs[i] + _, _ = app.ProcessTx(tx) + } else { + result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) - switch tracer := tracer.(type) { - case *vm.StructLogger: - return ðapi.ExecutionResult{ - Failed: err == nil, - ReturnValue: fmt.Sprintf("%x", result), - StructLogs: ethapi.FormatLogs(tracer.StructLogs()), - }, nil - case *tracers.Tracer: - return tracer.GetResult() - default: - return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) + switch tracer := tracer.(type) { + case *vm.StructLogger: + return ðapi.ExecutionResult{ + Failed: err == nil, + ReturnValue: fmt.Sprintf("%x", result), + StructLogs: ethapi.FormatLogs(tracer.StructLogs()), + }, nil + case *tracers.Tracer: + return tracer.GetResult() + default: + return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) + } + } + } + app.NextBlock() } + return nil, errors.New("counld not find transaction") } diff --git a/rpc/query_server.go b/rpc/query_server.go index 0043212aab..be684c813c 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -64,7 +64,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State - InMemoryApp(uint64) middleware.InMemoryApp + InMemoryApp(uint64, store.BlockStore) (middleware.InMemoryApp, error) } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -1126,7 +1126,11 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTra return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) } cfg := debug.DecTraceConfig(*config) - return debug.TraceTransaction(s.InMemoryApp(blockNumber), s.BlockStore, int64(blockNumber), txIndex, cfg) + memApp, err := s.InMemoryApp(blockNumber, s.BlockStore) + if err != nil { + return nil, err + } + return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), txIndex, cfg) } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { diff --git a/state/state.go b/state/state.go index 3c2643441d..2aae1db218 100644 --- a/state/state.go +++ b/state/state.go @@ -66,6 +66,21 @@ func NewStoreState( } } +func NewStoreState2( + ctx context.Context, + store store.KVStore, + block types.BlockHeader, + getValidatorSet GetValidatorSet, +) *StoreState { + return &StoreState{ + ctx: ctx, + store: store, + block: block, + validators: loom.NewValidatorSet(), + getValidatorSet: getValidatorSet, + } +} + func (s *StoreState) WithOnChainConfig(config *cctypes.Config) *StoreState { s.config = config return s diff --git a/store/iavlstore.go b/store/iavlstore.go index 253d6f5a00..8fd299e075 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -216,6 +216,19 @@ func (s *IAVLStore) GetSnapshot() Snapshot { } } +func (s *IAVLStore) VersionExists(version int64) bool { + return s.tree.VersionExists(version) +} + +func (s *IAVLStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + reader, err := newIAVLVersionReader(*s, version) + if err != nil { + return nil, err + } + splitStore := newSplitStore(reader, NewMemStore()) + return splitStore, nil +} + // NewIAVLStore creates a new IAVLStore. // maxVersions can be used to specify how many versions should be retained, if set to zero then // old versions will never been deleted. diff --git a/store/logstore.go b/store/logstore.go index 2e14a6b981..841437d185 100644 --- a/store/logstore.go +++ b/store/logstore.go @@ -129,3 +129,11 @@ func (s *LogStore) Prune() error { func (s *LogStore) GetSnapshot() Snapshot { return s.store.GetSnapshot() } + +func (s *LogStore) VersionExists(version int64) bool { + return s.store.VersionExists(version) +} + +func (s *LogStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + return s.store.RetrieveVersion(version) +} diff --git a/store/memstore.go b/store/memstore.go index badf02b59f..99c8984178 100644 --- a/store/memstore.go +++ b/store/memstore.go @@ -79,3 +79,11 @@ func (m *MemStore) Prune() error { func (m *MemStore) GetSnapshot() Snapshot { panic("not implemented") } + +func (m *MemStore) VersionExists(version int64) bool { + panic("not implemented") +} + +func (m *MemStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + panic("not implemented") +} diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index 50fa059163..4aa44d1dd4 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -354,3 +354,11 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { return ret } + +func (m *MultiWriterAppStore) VersionExists(version int64) bool { + return m.appStore.VersionExists(version) +} + +func (m *MultiWriterAppStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + return m.appStore.RetrieveVersion(version) +} diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 4e73cbad31..65a6ab8634 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -269,3 +269,11 @@ type pruningIAVLStoreSnapshot struct { func (s *pruningIAVLStoreSnapshot) Release() { // noop } + +func (m *PruningIAVLStore) VersionExists(version int64) bool { + return m.store.VersionExists(version) +} + +func (m *PruningIAVLStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + return m.store.RetrieveVersion(version) +} diff --git a/store/splitstore.go b/store/splitstore.go index 44f4671208..918f85f2f3 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -6,34 +6,34 @@ import ( ) type splitStore struct { + KVReader VersionedKVStore - diskStore VersionedKVStore - deleted map[string]bool + deleted map[string]bool } -func NewSplitStore(empty, full VersionedKVStore) VersionedKVStore { +func newSplitStore(full KVReader, empty VersionedKVStore) VersionedKVStore { return &splitStore{ + KVReader: full, VersionedKVStore: empty, - diskStore: full, deleted: make(map[string]bool), } } func (ss splitStore) Get(key []byte) []byte { - if ss.VersionedKVStore.Has(key) { - return ss.VersionedKVStore.Get(key) + if ss.KVReader.Has(key) { + return ss.KVReader.Get(key) } if ss.deleted[string(key)] { return nil } - return ss.diskStore.Get(key) + return ss.VersionedKVStore.Get(key) } func (ss splitStore) Range(prefix []byte) plugin.RangeData { - resultRange := ss.VersionedKVStore.Range(prefix) - diskRange := ss.diskStore.Range(prefix) + resultRange := ss.KVReader.Range(prefix) + diskRange := ss.VersionedKVStore.Range(prefix) for _, re := range diskRange { - if !ss.VersionedKVStore.Has(re.Key) && !ss.deleted[string(re.Key)] { + if !ss.KVReader.Has(re.Key) && !ss.deleted[string(re.Key)] { resultRange = append(resultRange, re) } } @@ -41,13 +41,13 @@ func (ss splitStore) Range(prefix []byte) plugin.RangeData { } func (ss splitStore) Has(key []byte) bool { - if ss.VersionedKVStore.Has(key) { + if ss.KVReader.Has(key) { return true } if ss.deleted[string(key)] { return false } - return ss.diskStore.Has(key) + return ss.VersionedKVStore.Has(key) } func (ss splitStore) Set(key, value []byte) { diff --git a/store/store.go b/store/store.go index f2aaae4320..1c5eb65d0c 100644 --- a/store/store.go +++ b/store/store.go @@ -54,6 +54,8 @@ type VersionedKVStore interface { // Delete old version of the store Prune() error GetSnapshot() Snapshot + VersionExists(int64) bool + RetrieveVersion(int64) (VersionedKVStore, error) } type cacheItem struct { diff --git a/store/version_reader.go b/store/version_reader.go new file mode 100644 index 0000000000..6929e6b965 --- /dev/null +++ b/store/version_reader.go @@ -0,0 +1,44 @@ +package store + +import ( + "github.com/loomnetwork/go-loom/plugin" + "github.com/pkg/errors" +) + +type iAVLVersionReader struct { + IAVLStore + version int64 +} + +func newIAVLVersionReader(store IAVLStore, version int64) (KVReader, error) { + if !store.tree.VersionExists(version) { + return nil, errors.Errorf("version %d missing", version) + } + return &iAVLVersionReader{ + IAVLStore: store, + version: version, + }, nil +} + +func (s *iAVLVersionReader) Get(key []byte) []byte { + value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) + return value +} + +// Range returns a range of keys +func (s *iAVLVersionReader) Range(prefix []byte) plugin.RangeData { + rangeData := s.IAVLStore.Range(prefix) + var rtv plugin.RangeData + for _, entry := range rangeData { + if s.Has(entry.Key) { + rtv = append(rtv, entry) + } + } + return rtv +} + +// Has checks if a key exists. +func (s *iAVLVersionReader) Has(key []byte) bool { + value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) + return value == nil +} diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 8f9575efff..2c1ab5c2ea 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -378,6 +378,14 @@ func (c *versionedCachingStore) GetSnapshot() Snapshot { ) } +func (c *versionedCachingStore) VersionExists(version int64) bool { + return c.VersionedKVStore.VersionExists(version) +} + +func (c *versionedCachingStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + return c.VersionedKVStore.RetrieveVersion(version) +} + // CachingStoreSnapshot is a read-only CachingStore with specified version type versionedCachingStoreSnapshot struct { Snapshot diff --git a/store/versioned_cachingstore_test.go b/store/versioned_cachingstore_test.go index 5345b77e5d..e0f7964c8d 100644 --- a/store/versioned_cachingstore_test.go +++ b/store/versioned_cachingstore_test.go @@ -58,6 +58,14 @@ func (m *MockStore) Prune() error { return nil } +func (m *MockStore) VersionExists(version int64) bool { + panic("not implemented") +} + +func (m *MockStore) RetrieveVersion(version int64) (VersionedKVStore, error) { + panic("not implemented") +} + func (m *MockStore) GetSnapshot() Snapshot { snapshotStore := make(map[string][]byte) for k, v := range m.storage { diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go index e189cca317..28b6ff6add 100644 --- a/txhandler/middleware/memory_app.go +++ b/txhandler/middleware/memory_app.go @@ -7,7 +7,9 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/loomnetwork/go-loom/builtin/types/chainconfig" - abci "github.com/tendermint/tendermint/abci/types" + "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" + ttypes "github.com/tendermint/tendermint/types" "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" @@ -17,12 +19,13 @@ import ( type InMemoryApp interface { ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) + NextBlock() + Height() int64 } type inMemoryApp struct { - lastBlockHeader abci.Header - curBlockHeader abci.Header - curBlockHash []byte + height int64 + blockstore store.BlockStore store store.VersionedKVStore txHandler txhandler.TxHandler receiptsVersion int32 @@ -32,9 +35,8 @@ type inMemoryApp struct { } func NewInMemoryApp( - lastBlockHeader abci.Header, - curBlockHeader abci.Header, - curBlockHash []byte, + height int64, + blockstore store.BlockStore, store store.VersionedKVStore, txHandler txhandler.TxHandler, receiptsVersion int32, @@ -43,9 +45,8 @@ func NewInMemoryApp( txHandlerFactory txhandler.TxHandlerFactory, ) InMemoryApp { return &inMemoryApp{ - lastBlockHeader: lastBlockHeader, - curBlockHeader: curBlockHeader, - curBlockHash: curBlockHash, + height: height, + blockstore: blockstore, store: store, txHandler: txHandler, receiptsVersion: receiptsVersion, @@ -55,6 +56,14 @@ func NewInMemoryApp( } } +func (ma *inMemoryApp) NextBlock() { + ma.height++ +} + +func (ma *inMemoryApp) Height() int64 { + return ma.height +} + func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) { return ma.processTx(txBytes, ma.txHandler) } @@ -62,11 +71,15 @@ func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, err func (ma *inMemoryApp) processTx(txBytes []byte, txHandler txhandler.TxHandler) (txhandler.TxHandlerResult, error) { splitStoreTx := store.WrapAtomic(ma.store).BeginTx() defer splitStoreTx.Rollback() - storeState := state.NewStoreState( + resultBlock, err := ma.blockstore.GetBlockByHeight(&ma.height) + if err != nil { + return txhandler.TxHandlerResult{}, errors.Errorf("retriving block for height %d", ma.height) + } + + storeState := state.NewStoreState2( context.Background(), splitStoreTx, - ma.curBlockHeader, - ma.curBlockHash, + blockHeaderFromHeader(resultBlock.BlockMeta.Header), ma.getValidatorSet, ).WithOnChainConfig(ma.config) return txHandler.ProcessTx(storeState, txBytes, false) @@ -97,3 +110,17 @@ func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { } return tracer, nil } + +func blockHeaderFromHeader(header ttypes.Header) types.BlockHeader { + return types.BlockHeader{ + ChainID: header.ChainID, + Height: header.Height, + Time: int64(header.Time.Unix()), + NumTxs: int32(header.NumTxs), + LastBlockID: types.BlockID{ + Hash: header.LastBlockID.Hash, + }, + ValidatorsHash: header.ValidatorsHash, + AppHash: header.AppHash, + } +} From 343e0395d667d2ce965558614ccf394b59930a67 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 27 Sep 2019 11:39:28 +0100 Subject: [PATCH 10/49] wip use correct chain id --- app.go | 8 ++-- auth/auth_test.go | 1 - auth/multi_chain_sigtx_middleware_test.go | 58 ++++++++++++----------- rpc/debug/trace.go | 54 ++++++++++----------- rpc/query_server.go | 2 +- store/block_store.go | 1 + store/version_reader.go | 2 +- txhandler/middleware/memory_app.go | 19 ++++++++ 8 files changed, 80 insertions(+), 65 deletions(-) diff --git a/app.go b/app.go index a07e448a50..ae566c9175 100644 --- a/app.go +++ b/app.go @@ -608,19 +608,19 @@ func (a *Application) ReadOnlyState() appstate.State { } func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStore) (middleware.InMemoryApp, error) { - startVersion := blockNumber - for ; a.Store.VersionExists(int64(startVersion)) || startVersion == 0; startVersion-- { + startVersion := int64(blockNumber) + for ; !a.Store.VersionExists(startVersion) || startVersion == 0; startVersion-- { } if startVersion == 0 { return nil, errors.Errorf("no saved version for height %d", blockNumber) } - startStore, err := a.Store.RetrieveVersion(int64(startVersion)) + startStore, err := a.Store.RetrieveVersion(startVersion) if err != nil { return nil, err } return middleware.NewInMemoryApp( - int64(startVersion), + startVersion, blockstore, startStore, a.TxHandler, diff --git a/auth/auth_test.go b/auth/auth_test.go index 374e171b41..5bf8b6488b 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -14,7 +14,6 @@ import ( "github.com/loomnetwork/go-loom/auth" "github.com/loomnetwork/go-loom/config" - "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/auth/keys" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" diff --git a/auth/multi_chain_sigtx_middleware_test.go b/auth/multi_chain_sigtx_middleware_test.go index 845324c19f..b1cbc6228d 100644 --- a/auth/multi_chain_sigtx_middleware_test.go +++ b/auth/multi_chain_sigtx_middleware_test.go @@ -12,19 +12,19 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/gogo/protobuf/proto" - loom "github.com/loomnetwork/go-loom" + "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/auth" amtypes "github.com/loomnetwork/go-loom/builtin/types/address_mapper" "github.com/loomnetwork/go-loom/common/evmcompat" goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" + gltypes "github.com/loomnetwork/go-loom/types" "github.com/loomnetwork/go-loom/vm" sha3 "github.com/miguelmota/go-solidity-sha3" "github.com/pkg/errors" "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/address_mapper" "github.com/loomnetwork/loomchain/features" @@ -35,7 +35,7 @@ import ( const ( callId = uint32(2) - sequence = uint64(4) + seq = uint64(4) defaultLoomChainId = "default" ) @@ -53,11 +53,13 @@ var ( func TestSigning(t *testing.T) { pk, err := crypto.GenerateKey() + fmt.Printf("private key %x\n", pk) require.NoError(t, err) err = crypto.SaveECDSA("newpk", pk) require.NoError(t, err) privateKey, err := crypto.HexToECDSA(ethPrivateKey) + fmt.Printf("private key %x\n", privateKey) require.NoError(t, err) publicKey := crypto.FromECDSAPub(&privateKey.PublicKey) require.NoError(t, err) @@ -182,14 +184,14 @@ func TestEthAddressMappingVerification(t *testing.T) { ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) - chains := map[string]ChainConfig{ + chains := map[string]keys.ChainConfig{ "default": { - TxType: LoomSignedTxType, - AccountType: NativeAccountType, + TxType: keys.LoomSignedTxType, + AccountType: keys.NativeAccountType, }, "eth": { - TxType: EthereumSignedTxType, - AccountType: MappedAccountType, + TxType: keys.EthereumSignedTxType, + AccountType: keys.MappedAccountType, }, } tmx := NewMultiChainSignatureTxMiddleware( @@ -251,14 +253,14 @@ func TestBinanceAddressMappingVerification(t *testing.T) { ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) - chains := map[string]ChainConfig{ + chains := map[string]keys.ChainConfig{ "default": { - TxType: LoomSignedTxType, - AccountType: NativeAccountType, + TxType: keys.LoomSignedTxType, + AccountType: keys.NativeAccountType, }, "binance": { - TxType: BinanceSignedTxType, - AccountType: MappedAccountType, + TxType: keys.BinanceSignedTxType, + AccountType: keys.MappedAccountType, }, } tmx := NewMultiChainSignatureTxMiddleware( @@ -321,22 +323,22 @@ func TestChainIdVerification(t *testing.T) { ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) - chains := map[string]ChainConfig{ + chains := map[string]keys.ChainConfig{ "default": { - TxType: LoomSignedTxType, - AccountType: NativeAccountType, + TxType: keys.LoomSignedTxType, + AccountType: keys.NativeAccountType, }, "eth": { - TxType: EthereumSignedTxType, - AccountType: NativeAccountType, + TxType: keys.EthereumSignedTxType, + AccountType: keys.NativeAccountType, }, "tron": { - TxType: TronSignedTxType, - AccountType: NativeAccountType, + TxType: keys.TronSignedTxType, + AccountType: keys.NativeAccountType, }, "binance": { - TxType: BinanceSignedTxType, - AccountType: NativeAccountType, + TxType: keys.BinanceSignedTxType, + AccountType: keys.NativeAccountType, }, } tmx := NewMultiChainSignatureTxMiddleware( @@ -381,7 +383,7 @@ func TestChainIdVerification(t *testing.T) { require.NoError(t, err) } -func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, signedTx []byte, ctx context.Context) (loomchain.TxHandlerResult, error) { +func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, signedTx []byte, ctx context.Context) (txhandler.TxHandlerResult, error) { return ttm.ProcessTx(state.WithContext(ctx), signedTx, func(state appstate.State, txBytes []byte, isCheckTx bool) (res txhandler.TxHandlerResult, err error) { @@ -390,7 +392,7 @@ func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.St return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx txhandler.Transaction + var tx gltypes.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } @@ -418,7 +420,7 @@ func mockEd25519SignedTx(t *testing.T, key string) []byte { ChainID: defaultLoomChainId, Local: loom.LocalAddressFromPublicKey(signer.PublicKey()), } - nonceTx := mockNonceTx(t, addr, sequence) + nonceTx := mockNonceTx(t, addr, seq) signedTx := auth.SignTx(signer, nonceTx) marshalledSignedTx, err := proto.Marshal(signedTx) @@ -431,7 +433,7 @@ func mockSignedTx(t *testing.T, chainID string, signer auth.Signer) []byte { require.NoError(t, err) ethLocalAdr, err := loom.LocalAddressFromHexString(crypto.PubkeyToAddress(*pubKey).Hex()) require.NoError(t, err) - nonceTx := mockNonceTx(t, loom.Address{ChainID: chainID, Local: ethLocalAdr}, sequence) + nonceTx := mockNonceTx(t, loom.Address{ChainID: chainID, Local: ethLocalAdr}, seq) signedTx := auth.SignTx(signer, nonceTx) marshalledSignedTx, err := proto.Marshal(signedTx) @@ -442,7 +444,7 @@ func mockSignedTx(t *testing.T, chainID string, signer auth.Signer) []byte { func mockBinanceSignedTx(t *testing.T, chainID string, signer auth.Signer) []byte { foreignLocalAddr, err := loom.LocalAddressFromHexString(evmcompat.BitcoinAddress(signer.PublicKey()).Hex()) require.NoError(t, err) - nonceTx := mockNonceTx(t, loom.Address{ChainID: chainID, Local: foreignLocalAddr}, sequence) + nonceTx := mockNonceTx(t, loom.Address{ChainID: chainID, Local: foreignLocalAddr}, seq) signedTx := auth.SignTx(signer, nonceTx) marshalledSignedTx, err := proto.Marshal(signedTx) @@ -463,7 +465,7 @@ func mockNonceTx(t *testing.T, from loom.Address, sequence uint64) []byte { From: from.MarshalPB(), }) require.NoError(t, err) - tx, err := proto.Marshal(&txhandler.Transaction{ + tx, err := proto.Marshal(&gltypes.Transaction{ Id: callId, Data: messageTx, }) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index d0c43c59b1..87d4d85f7f 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -18,37 +18,31 @@ import ( func TraceTransaction( app middleware.InMemoryApp, blockstore store.BlockStore, - blockNumber int64, - txIndex uint64, + blockNumber, txIndex int64, config eth.TraceConfig, ) (interface{}, error) { - for height := app.Height(); height <= blockNumber; height++ { - block, err := blockstore.GetBlockByHeight(&blockNumber) - if err != nil { - return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) - } - for i := uint64(0); i < txIndex; i++ { - if height != blockNumber || i != txIndex { - tx := block.Block.Data.Txs[i] - _, _ = app.ProcessTx(tx) - } else { - result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) - - switch tracer := tracer.(type) { - case *vm.StructLogger: - return ðapi.ExecutionResult{ - Failed: err == nil, - ReturnValue: fmt.Sprintf("%x", result), - StructLogs: ethapi.FormatLogs(tracer.StructLogs()), - }, nil - case *tracers.Tracer: - return tracer.GetResult() - default: - return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) - } - } - } - app.NextBlock() + if err := app.RunUpTo(blockNumber, txIndex); err != nil { + return nil, err } - return nil, errors.New("counld not find transaction") + + block, err := blockstore.GetBlockByHeight(&blockNumber) + if err != nil { + return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) + } + + result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) + + switch tracer := tracer.(type) { + case *vm.StructLogger: + return ðapi.ExecutionResult{ + Failed: err == nil, + ReturnValue: fmt.Sprintf("%x", result), + StructLogs: ethapi.FormatLogs(tracer.StructLogs()), + }, nil + case *tracers.Tracer: + return tracer.GetResult() + default: + return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) + } + } diff --git a/rpc/query_server.go b/rpc/query_server.go index be684c813c..83fd821e1d 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -1130,7 +1130,7 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTra if err != nil { return nil, err } - return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), txIndex, cfg) + return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), int64(txIndex), cfg) } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { diff --git a/store/block_store.go b/store/block_store.go index 9a9acda884..d4e578a91c 100644 --- a/store/block_store.go +++ b/store/block_store.go @@ -230,6 +230,7 @@ func (s *TendermintBlockStore) GetBlockByHeight(height *int64) (*ctypes.ResultBl LastBlockID: blockResult.Block.Header.LastBlockID, Time: blockResult.Block.Header.Time, ProposerAddress: blockResult.Block.Header.ProposerAddress, + ChainID: blockResult.Block.Header.ChainID, } blockMeta := types.BlockMeta{ BlockID: blockResult.BlockMeta.BlockID, diff --git a/store/version_reader.go b/store/version_reader.go index 6929e6b965..7b077f9b16 100644 --- a/store/version_reader.go +++ b/store/version_reader.go @@ -40,5 +40,5 @@ func (s *iAVLVersionReader) Range(prefix []byte) plugin.RangeData { // Has checks if a key exists. func (s *iAVLVersionReader) Has(key []byte) bool { value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) - return value == nil + return value != nil } diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go index 28b6ff6add..cb0c140576 100644 --- a/txhandler/middleware/memory_app.go +++ b/txhandler/middleware/memory_app.go @@ -19,6 +19,7 @@ import ( type InMemoryApp interface { ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) + RunUpTo(height, index int64) error NextBlock() Height() int64 } @@ -64,6 +65,24 @@ func (ma *inMemoryApp) Height() int64 { return ma.height } +func (ma *inMemoryApp) RunUpTo(height, index int64) error { + for h := ma.Height(); h <= height; h++ { + block, err := ma.blockstore.GetBlockByHeight(&h) + if err != nil { + return errors.Wrapf(err, "getting block information at height %v", height) + } + for i := 0; i < len(block.Block.Data.Txs); i++ { + if h != height || i != int(index) { + _, _ = ma.ProcessTx(block.Block.Data.Txs[i]) + } else { + return nil + } + } + ma.NextBlock() + } + return errors.Errorf("cannot find transaction at height %d index %d", height, index) +} + func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) { return ma.processTx(txBytes, ma.txHandler) } From 3d71a4f2d75279e54db8b72c20e8a1016826fff4 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 27 Sep 2019 15:38:47 +0100 Subject: [PATCH 11/49] move nocetxhandler into loom.go --- auth/auth.go | 21 ++-- auth/auth_test.go | 32 ++--- cmd/loom/loom.go | 168 +-------------------------- txhandler/factory/handler_factory.go | 16 ++- 4 files changed, 46 insertions(+), 191 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 7f3a9da7b3..78291dfae0 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -174,18 +174,21 @@ func (n *NonceHandler) IncNonce(state appstate.State, return nil } -var NonceTxHandler = NonceHandler{nonceCache: make(map[string]uint64), lastHeight: 0} - -var NonceTxPostNonceMiddleware = txhandler.PostCommitMiddlewareFunc(NonceTxHandler.IncNonce) +func NewNonceHandler() *NonceHandler { + return &NonceHandler{nonceCache: make(map[string]uint64), lastHeight: 0} +} -var NonceTxMiddleware = func(kvStore store.KVStore) txhandler.TxMiddlewareFunc { - nonceTxMiddleware := func( +func GetNonceTxMiddleware(kvStore store.KVStore, nonceTxHandler *NonceHandler) txhandler.TxMiddlewareFunc { + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, next txhandler.TxHandlerFunc, isCheckTx bool, - ) (txhandler.TxHandlerResult, error) { - return NonceTxHandler.Nonce(state, kvStore, txBytes, next, isCheckTx) - } - return txhandler.TxMiddlewareFunc(nonceTxMiddleware) + ) (res txhandler.TxHandlerResult, err error) { + return nonceTxHandler.Nonce(state, kvStore, txBytes, next, isCheckTx) + }) +} + +func GetNonceTxPostNonceMiddleware(nonceTxHandler *NonceHandler) txhandler.PostCommitMiddlewareFunc { + return nonceTxHandler.IncNonce } diff --git a/auth/auth_test.go b/auth/auth_test.go index 5bf8b6488b..0abcb3ea81 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -20,6 +20,10 @@ import ( "github.com/loomnetwork/loomchain/txhandler" ) +var nonceTxHandler = NonceHandler{nonceCache: make(map[string]uint64), lastHeight: 0} + +var nonceTxPostNonceMiddleware = txhandler.PostCommitMiddlewareFunc(nonceTxHandler.IncNonce) + func TestSignatureTxMiddleware(t *testing.T) { origBytes := []byte("hello") _, privKey, err := ed25519.GenerateKey(nil) @@ -69,13 +73,13 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { kvStore := store.NewMemStore() state := appstate.NewStoreState(ctx, kvStore, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) - _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes, + _, err = nonceTxHandler.Nonce(state, kvStore, nonceTxBytes, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) + nonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) //State is reset on every run ctx2 := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) @@ -84,13 +88,13 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { _ = context.WithValue(ctx2, keys.ContextKeyCheckTx, true) //If we get the same sequence number in same block we should get an error - _, err = NonceTxHandler.Nonce(state2, kvStore2, nonceTxBytes, + _, err = nonceTxHandler.Nonce(state2, kvStore2, nonceTxBytes, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, true, ) require.Errorf(t, err, "sequence number does not match") - // NonceTxPostNonceMiddleware shouldnt get called on an error + // nonceTxPostNonceMiddleware shouldnt get called on an error //State is reset on every run ctx3 := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) @@ -99,13 +103,13 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { _ = context.WithValue(ctx3, keys.ContextKeyCheckTx, true) //If we get to tx with incrementing sequence numbers we should be fine in the same block - _, err = NonceTxHandler.Nonce(state3, kvStore3, nonceTxBytes2, + _, err = nonceTxHandler.Nonce(state3, kvStore3, nonceTxBytes2, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, true, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes2, txhandler.TxHandlerResult{}, nil, false) + nonceTxPostNonceMiddleware(state, nonceTxBytes2, txhandler.TxHandlerResult{}, nil, false) //Try a deliverTx at same height it should be fine ctx3Dx := context.WithValue(context.Background(), keys.ContextKeyOrigin, origin) @@ -113,13 +117,13 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { state3Dx := appstate.NewStoreState(ctx3Dx, kvStore3Dx, abci.Header{Height: 27}, nil, nil).WithOnChainConfig(cfg) _ = context.WithValue(ctx3Dx, keys.ContextKeyCheckTx, true) - _, err = NonceTxHandler.Nonce(state3Dx, kvStore3Dx, nonceTxBytes, + _, err = nonceTxHandler.Nonce(state3Dx, kvStore3Dx, nonceTxBytes, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) + nonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) ///--------------increase block height should kill cache //State is reset on every run @@ -127,13 +131,13 @@ func TestSignatureTxMiddlewareMultipleTxSameBlock(t *testing.T) { kvStore4 := store.NewMemStore() state4 := appstate.NewStoreState(ctx4, kvStore4, abci.Header{Height: 28}, nil, nil).WithOnChainConfig(cfg) //If we get to tx with incrementing sequence numbers we should be fine in the same block - _, err = NonceTxHandler.Nonce(state4, kvStore4, nonceTxBytes, + _, err = nonceTxHandler.Nonce(state4, kvStore4, nonceTxBytes, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, true, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) + nonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) } func TestRevertedTxNonceMiddleware(t *testing.T) { @@ -173,18 +177,18 @@ func TestRevertedTxNonceMiddleware(t *testing.T) { require.Equal(t, uint64(0), currentNonce) // Send a successful tx - _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes, + _, err = nonceTxHandler.Nonce(state, kvStore, nonceTxBytes, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, nil }, false, ) require.Nil(t, err) - NonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) + nonceTxPostNonceMiddleware(state, nonceTxBytes, txhandler.TxHandlerResult{}, nil, false) storeTx.Commit() storeTx.Rollback() // Send a failed tx, nonce should increase even though the transaction is reverted - _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes2, + _, err = nonceTxHandler.Nonce(state, kvStore, nonceTxBytes2, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, errors.New("EVM transaction reverted") }, false, @@ -205,7 +209,7 @@ func TestRevertedTxNonceMiddleware(t *testing.T) { require.NoError(t, err) // Send another failed tx, nonce should not increment because the transaction reverted - _, err = NonceTxHandler.Nonce(state, kvStore, nonceTxBytes3, + _, err = nonceTxHandler.Nonce(state, kvStore, nonceTxBytes3, func(_ appstate.State, txBytes []byte, isCheckTx bool) (txhandler.TxHandlerResult, error) { return txhandler.TxHandlerResult{}, errors.New("EVM transaction reverted") }, false, diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index e13081f470..7a0ac253a0 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -30,26 +30,20 @@ import ( "github.com/loomnetwork/go-loom/util" "github.com/prometheus/client_golang/prometheus" + "github.com/pkg/errors" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cobra" + "golang.org/x/crypto/ed25519" + "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/abci/backend" - "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/builtin/plugins/dposv2" "github.com/loomnetwork/loomchain/builtin/plugins/dposv3" plasmaConfig "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/config" plasmaOracle "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/oracle" "github.com/loomnetwork/loomchain/features" - "github.com/loomnetwork/loomchain/migrations" "github.com/loomnetwork/loomchain/receipts/leveldb" - "github.com/loomnetwork/loomchain/throttle" - "github.com/loomnetwork/loomchain/tx_handler" - "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/txhandler/factory" - "github.com/loomnetwork/loomchain/txhandler/middleware" - - "github.com/pkg/errors" - stdprometheus "github.com/prometheus/client_golang/prometheus" - "github.com/spf13/cobra" - "golang.org/x/crypto/ed25519" "github.com/loomnetwork/loomchain/chainconfig" chaincfgcmd "github.com/loomnetwork/loomchain/cmd/loom/chainconfig" @@ -1034,158 +1028,6 @@ func deployContract( return nil } -func txMiddleWare( - cfg config.Config, - vmManager vm.Manager, - chainID string, - appStore store.VersionedKVStore, -) ([]txhandler.TxMiddleware, error) { - txMiddleWare := []txhandler.TxMiddleware{ - txhandler.LogTxMiddleware, - txhandler.RecoveryTxMiddleware, - } - - txMiddleWare = append(txMiddleWare, auth.NewChainConfigMiddleware( - cfg.Auth, - getContractStaticCtx("addressmapper", vmManager), - )) - - if cfg.Karma.Enabled { - txMiddleWare = append(txMiddleWare, throttle.GetKarmaMiddleWare( - cfg.Karma.Enabled, - cfg.Karma.MaxCallCount, - cfg.Karma.SessionDuration, - getContractCtx("karma", vmManager), - )) - } - - if cfg.TxLimiter.Enabled { - txMiddleWare = append(txMiddleWare, middleware.NewTxLimiterMiddleware(cfg.TxLimiter)) - } - - if cfg.ContractTxLimiter.Enabled { - udwCtxFactory := getContractCtx("user-deployer-whitelist", vmManager) - txMiddleWare = append( - txMiddleWare, middleware.NewContractTxLimiterMiddleware(cfg.ContractTxLimiter, udwCtxFactory), - ) - } - - if cfg.DeployerWhitelist.ContractEnabled { - dwMiddleware, err := middleware.NewDeployerWhitelistMiddleware(getContractCtx("deployerwhitelist", vmManager)) - if err != nil { - return nil, err - } - txMiddleWare = append(txMiddleWare, dwMiddleware) - - } - - txMiddleWare = append(txMiddleWare, auth.NonceTxMiddleware(appStore)) - - if cfg.GoContractDeployerWhitelist.Enabled { - goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) - if err != nil { - return nil, errors.Wrapf(err, "getting list of users allowed go deploys") - } - txMiddleWare = append(txMiddleWare, throttle.GetGoDeployTxMiddleWare(goDeployers)) - } - - txMiddleWare = append(txMiddleWare, txhandler.NewInstrumentingTxMiddleware()) - - return txMiddleWare, nil -} - -func router( - cfg config.Config, - vmManager vm.Manager, - createRegistry registry.RegistryFactoryFunc, -) txhandler.TxHandler { - router := middleware.NewTxRouter() - isEvmTx := func(txID uint32, _ appstate.State, txBytes []byte, isCheckTx bool) bool { - var msg vm.MessageTx - err := proto.Unmarshal(txBytes, &msg) - if err != nil { - return false - } - - switch txID { - case 1: - var tx vm.DeployTx - err = proto.Unmarshal(msg.Data, &tx) - if err != nil { - // In case of error, let's give safest response, - // let's TxHandler down the line, handle it. - return false - } - return tx.VmType == vm.VMType_EVM - case 2: - var tx vm.CallTx - err = proto.Unmarshal(msg.Data, &tx) - if err != nil { - // In case of error, let's give safest response, - // let's TxHandler down the line, handle it. - return false - } - return tx.VmType == vm.VMType_EVM - case 3: - return false - default: - return false - } - } - - deployTxHandler := &vm.DeployTxHandler{ - Manager: &vmManager, - CreateRegistry: createRegistry, - AllowNamedEVMContracts: cfg.AllowNamedEvmContracts, - } - - callTxHandler := &vm.CallTxHandler{ - Manager: &vmManager, - } - - migrationTxHandler := &tx_handler.MigrationTxHandler{ - Manager: &vmManager, - CreateRegistry: createRegistry, - Migrations: map[int32]tx_handler.MigrationFunc{ - 1: migrations.DPOSv3Migration, - 2: migrations.GatewayMigration, - 3: migrations.GatewayMigration, - }, - } - - router.HandleDeliverTx(1, middleware.GeneratePassthroughRouteHandler(deployTxHandler)) - router.HandleDeliverTx(2, middleware.GeneratePassthroughRouteHandler(callTxHandler)) - router.HandleDeliverTx(3, middleware.GeneratePassthroughRouteHandler(migrationTxHandler)) - - // TODO: Write this in more elegant way - router.HandleCheckTx(1, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, deployTxHandler)) - router.HandleCheckTx(2, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, callTxHandler)) - router.HandleCheckTx(3, middleware.GenerateConditionalRouteHandler(isEvmTx, txhandler.NoopTxHandler, migrationTxHandler)) - - return router -} - -func postCommitMiddleWAre(cfg config.Config, vmManager vm.Manager) ([]txhandler.PostCommitMiddleware, error) { - postCommitMiddlewares := []txhandler.PostCommitMiddleware{ - txhandler.LogPostCommitMiddleware, - } - - if cfg.UserDeployerWhitelist.ContractEnabled { - udwContractFactory := getContractCtx("user-deployer-whitelist", vmManager) - evmDeployRecorderMiddleware, err := middleware.NewEVMDeployRecorderPostCommitMiddleware(udwContractFactory) - if err != nil { - return nil, err - } - postCommitMiddlewares = append(postCommitMiddlewares, evmDeployRecorderMiddleware) - } - - // We need to make sure nonce post commit middleware is last as - // it doesn't pass control to other middlewares after it. - postCommitMiddlewares = append(postCommitMiddlewares, auth.NonceTxPostNonceMiddleware) - - return postCommitMiddlewares, nil -} - func getContractCtx(pluginName string, vmManager vm.Manager) func(state appstate.State) (contractpb.Context, error) { return func(state appstate.State) (contractpb.Context, error) { pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 88bfaf575a..ffae9635a9 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -48,12 +48,13 @@ type txHandleFactory struct { func (f txHandleFactory) TxHandler(tracer ethvm.Tracer, metrics bool) (txhandler.TxHandler, error) { vmManager := createVmManager(f.vmManager, tracer) + nonceTxHandler := auth.NewNonceHandler() - txMiddleware, err := txMiddleWare(f.cfg, vmManager, f.chainID, f.store, metrics) + txMiddleware, err := txMiddleWare(f.cfg, vmManager, nonceTxHandler, f.chainID, f.store, metrics) if err != nil { return nil, err } - postCommitMiddlewares, err := postCommitMiddleWAre(f.cfg, vmManager) + postCommitMiddlewares, err := postCommitMiddleWAre(f.cfg, vmManager, nonceTxHandler) if err != nil { return nil, err } @@ -80,6 +81,7 @@ func createVmManager(vmManager *vm.Manager, tracer ethvm.Tracer) vm.Manager { func txMiddleWare( cfg config.Config, vmManager vm.Manager, + nonceTxHandler *auth.NonceHandler, chainID string, appStore store.VersionedKVStore, metrics bool, @@ -123,7 +125,7 @@ func txMiddleWare( } - txMiddleWare = append(txMiddleWare, auth.NonceTxMiddleware(appStore)) + txMiddleWare = append(txMiddleWare, auth.GetNonceTxMiddleware(appStore, nonceTxHandler)) if cfg.GoContractDeployerWhitelist.Enabled { goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) @@ -211,7 +213,11 @@ func router( return router } -func postCommitMiddleWAre(cfg config.Config, vmManager vm.Manager) ([]txhandler.PostCommitMiddleware, error) { +func postCommitMiddleWAre( + cfg config.Config, + vmManager vm.Manager, + nonceTxHandler *auth.NonceHandler, +) ([]txhandler.PostCommitMiddleware, error) { postCommitMiddlewares := []txhandler.PostCommitMiddleware{ txhandler.LogPostCommitMiddleware, } @@ -227,7 +233,7 @@ func postCommitMiddleWAre(cfg config.Config, vmManager vm.Manager) ([]txhandler. // We need to make sure nonce post commit middleware is last as // it doesn't pass control to other middlewares after it. - postCommitMiddlewares = append(postCommitMiddlewares, auth.NonceTxPostNonceMiddleware) + postCommitMiddlewares = append(postCommitMiddlewares, auth.GetNonceTxPostNonceMiddleware(nonceTxHandler)) return postCommitMiddlewares, nil } From 7665fdf545b367c0fe9802ca0423d6572afddd30 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 30 Sep 2019 10:53:04 +0100 Subject: [PATCH 12/49] evm debug --- app.go | 7 +++++-- cmd/loom/loom.go | 7 +++---- evm/evm.go | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app.go b/app.go index ae566c9175..24a67dd3ad 100644 --- a/app.go +++ b/app.go @@ -608,8 +608,11 @@ func (a *Application) ReadOnlyState() appstate.State { } func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStore) (middleware.InMemoryApp, error) { - startVersion := int64(blockNumber) - for ; !a.Store.VersionExists(startVersion) || startVersion == 0; startVersion-- { + startVersion := int64(blockNumber) - 1 + if startVersion < 0 { + return nil, errors.Errorf("invalid block number %d", blockNumber) + } + for ; !a.Store.VersionExists(startVersion) || startVersion <= int64(0); startVersion-- { } if startVersion == 0 { return nil, errors.Errorf("no saved version for height %d", blockNumber) diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index 7a0ac253a0..cb4bb0177f 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -15,10 +15,6 @@ import ( "syscall" "time" - "github.com/loomnetwork/go-loom/plugin/contractpb" - "github.com/prometheus/client_golang/prometheus/push" - "github.com/tendermint/tendermint/libs/db" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" @@ -27,8 +23,11 @@ import ( "github.com/loomnetwork/go-loom/cli" "github.com/loomnetwork/go-loom/client" "github.com/loomnetwork/go-loom/crypto" + "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/util" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" + "github.com/tendermint/tendermint/libs/db" "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" diff --git a/evm/evm.go b/evm/evm.go index ed24a7110d..002d6de560 100644 --- a/evm/evm.go +++ b/evm/evm.go @@ -306,7 +306,7 @@ func defaultChainConfig(enableConstantinople bool) params.ChainConfig { } func createVmConfig(evmDebuggingEnabled bool, tracer vm.Tracer) (vm.Config, error) { - if evmDebuggingEnabled { + if evmDebuggingEnabled || tracer != nil { log.Error("WARNING!!!! EVM Debug mode enabled, do NOT run this on a production server!!!") } if tracer == nil { @@ -320,7 +320,7 @@ func createVmConfig(evmDebuggingEnabled bool, tracer vm.Tracer) (vm.Config, erro } return vm.Config{ // Debug enabled debugging Interpreter options - Debug: evmDebuggingEnabled, + Debug: evmDebuggingEnabled || (tracer != nil), // Tracer is the op code logger Tracer: tracer, // NoRecursion disabled Interpreter call, callcode, From 96b74f619b49fbd6f8aadc96054563086504184e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 30 Sep 2019 12:58:08 +0100 Subject: [PATCH 13/49] move tx_handler --- txhandler/factory/handler_factory.go | 6 +++--- {tx_handler => txhandler/migration}/migration_tx_handler.go | 2 +- .../migration}/migration_tx_handler_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename {tx_handler => txhandler/migration}/migration_tx_handler.go (99%) rename {tx_handler => txhandler/migration}/migration_tx_handler_test.go (99%) diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index ffae9635a9..8ea80cf32a 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -16,9 +16,9 @@ import ( "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" "github.com/loomnetwork/loomchain/throttle" - "github.com/loomnetwork/loomchain/tx_handler" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/txhandler/middleware" + "github.com/loomnetwork/loomchain/txhandler/migration" "github.com/loomnetwork/loomchain/vm" ) @@ -191,10 +191,10 @@ func router( Manager: &vmManager, } - migrationTxHandler := &tx_handler.MigrationTxHandler{ + migrationTxHandler := &migration.MigrationTxHandler{ Manager: &vmManager, CreateRegistry: createRegistry, - Migrations: map[int32]tx_handler.MigrationFunc{ + Migrations: map[int32]migration.MigrationFunc{ 1: migrations.DPOSv3Migration, 2: migrations.GatewayMigration, 3: migrations.GatewayMigration, diff --git a/tx_handler/migration_tx_handler.go b/txhandler/migration/migration_tx_handler.go similarity index 99% rename from tx_handler/migration_tx_handler.go rename to txhandler/migration/migration_tx_handler.go index 922f087934..cf5b3bb298 100644 --- a/tx_handler/migration_tx_handler.go +++ b/txhandler/migration/migration_tx_handler.go @@ -1,4 +1,4 @@ -package tx_handler +package migration import ( "bytes" diff --git a/tx_handler/migration_tx_handler_test.go b/txhandler/migration/migration_tx_handler_test.go similarity index 99% rename from tx_handler/migration_tx_handler_test.go rename to txhandler/migration/migration_tx_handler_test.go index ba54ec03a3..0ceee73951 100644 --- a/tx_handler/migration_tx_handler_test.go +++ b/txhandler/migration/migration_tx_handler_test.go @@ -1,4 +1,4 @@ -package tx_handler +package migration import ( "context" From 63368c5a6b655cf4409871cc4edf0058743bbe11 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 10 Oct 2019 18:16:33 +0100 Subject: [PATCH 14/49] merge master changes --- txhandler/factory/handler_factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 8ea80cf32a..153c0ba646 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -125,7 +125,7 @@ func txMiddleWare( } - txMiddleWare = append(txMiddleWare, auth.GetNonceTxMiddleware(appStore, nonceTxHandler)) + txMiddleWare = append(txMiddleWare, nonceTxHandler.TxMiddleware(appStore)) if cfg.GoContractDeployerWhitelist.Enabled { goDeployers, err := cfg.GoContractDeployerWhitelist.DeployerAddresses(chainID) @@ -233,7 +233,7 @@ func postCommitMiddleWAre( // We need to make sure nonce post commit middleware is last as // it doesn't pass control to other middlewares after it. - postCommitMiddlewares = append(postCommitMiddlewares, auth.GetNonceTxPostNonceMiddleware(nonceTxHandler)) + postCommitMiddlewares = append(postCommitMiddlewares, nonceTxHandler.PostCommitMiddleware()) return postCommitMiddlewares, nil } From 91f266f7d2e0a7d99525d2fcbcbf07f2a4a70e75 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 14 Oct 2019 16:22:05 +0100 Subject: [PATCH 15/49] newMultiWriterVersionReader, unit test --- e2e/tests/truffle/test/DebugFunctions.js | 146 +++++++++++++++++++++++ rpc/debug/jsonrpc_conversion.go | 10 +- rpc/debug/trace.go | 1 + rpc/query_server.go | 5 +- store/multi_writer_app_store.go | 7 +- store/splitstore.go | 20 ++-- store/store_test.go | 26 +++- store/version_reader.go | 55 +++++++++ 8 files changed, 252 insertions(+), 18 deletions(-) create mode 100644 e2e/tests/truffle/test/DebugFunctions.js diff --git a/e2e/tests/truffle/test/DebugFunctions.js b/e2e/tests/truffle/test/DebugFunctions.js new file mode 100644 index 0000000000..b2b3677d85 --- /dev/null +++ b/e2e/tests/truffle/test/DebugFunctions.js @@ -0,0 +1,146 @@ +const Web3 = require('web3'); +const fs = require('fs'); +const path = require('path'); +const EthereumTx = require('ethereumjs-tx').Transaction; +const { getLoomEvmTxHash } = require('./helpers'); + +const { + SpeculativeNonceTxMiddleware, SignedTxMiddleware, Client, + LocalAddress, CryptoUtils, LoomProvider +} = require('loom-js'); + +const TxHashTestContract = artifacts.require('TxHashTestContract'); +const NonceTestContract = artifacts.require('NonceTestContract'); +const SimpleStore = artifacts.require('SimpleStore'); + +// Requires receipts:v3.3 to be enabled, and receipts:v3.4 not to be, but the new tx hash algo needs +// more review & testing before we can release it so skipping this test for now. +contract('TxHashTestContract', async (accounts) => { + let contract, nonceContract, fromAddr, nodeAddr, txHashTestContract, nonceTestContract, web3, web3js; + + beforeEach(async () => { + nodeAddr = fs.readFileSync(path.join(process.env.CLUSTER_DIR, '0', 'node_rpc_addr'), 'utf-8').trim() + const chainID = 'default'; + const writeUrl = `ws://${nodeAddr}/websocket`; + const readUrl = `ws://${nodeAddr}/queryws`; + + const privateKey = CryptoUtils.generatePrivateKey(); + const publicKey = CryptoUtils.publicKeyFromPrivateKey(privateKey); + + fromAddr = LocalAddress.fromPublicKey(publicKey); + const from = fromAddr.toString(); + + var client = new Client(chainID, writeUrl, readUrl); + client.on('error', msg => { + console.error('Error on connect to client', msg); + console.warn('Please verify if loom cluster is running') + }); + const setupMiddlewareFn = function(client, privateKey) { + const publicKey = CryptoUtils.publicKeyFromPrivateKey(privateKey); + return [new SpeculativeNonceTxMiddleware(publicKey, client), new SignedTxMiddleware(privateKey)] + }; + var loomProvider = new LoomProvider(client, privateKey, setupMiddlewareFn); + + web3 = new Web3(loomProvider); + web3js = new Web3(new Web3.providers.HttpProvider(`http://${nodeAddr}/eth`)); + + + //nonceTestContract = await SimpleStore.deployed(); + //nonceContract = new web3.eth.Contract(SimpleStore._json.abi, nonceTestContract.address, {from}); + + txHashTestContract = await TxHashTestContract.deployed(); + contract = new web3.eth.Contract(TxHashTestContract._json.abi, txHashTestContract.address, {from}); + }); + + it('Test debug_traceTransaction', async () => { + console.log("piers contract.methods|", contract.methods); + console.log("piers contract.method.set|", contract.methods.set); + + // console.log("piers nonceTestContract.methods|", nonceTestContract.methods); + // console.log("piers nonceTestContract.methods.setValue|", nonceTestContract.methods.setValue); + // console.log("piers nonceTestContract.methods.setValue|", nonceTestContract.methods.setValue(uint256)); + try { +/* + const nonceTxResult = await nonceTestContract.methods.set(1111).send(); + console.log("piers nonceTxResult txResult.tx|", nonceTxResult.tx); + console.log("piers nonceTestContract txResult.transactionHash|", nonceTxResult.transactionHash); + await web3js.currentProvider.send({ + method: "debug_traceTransaction", + params: [nonceTxResult.transactionHash], + jsonrpc: "2.0", + id: new Date().getTime() + }, function (error, result) { + console.log("piers!!!!!!!!!nonceTxResult debug_traceTransaction sendResult|", result, "error", error) + console.log("piersn onceTxResult failed|", result.failed); + console.log("piers nonceTxResult structLogs|", result.structLogs); + //assert.equal(true, result === result); + assert.equal(false, result.failed) + }); +*/ + const txResult = await contract.methods.set(1111).send(); + console.log("piers txResult.tx|", txResult.tx); + console.log("piers txResult.transactionHash|", txResult.transactionHash); + + await web3js.currentProvider.send({ + method: "debug_traceTransaction", + params: [txResult.transactionHash,{"disableStorage":true,"disableMemory":false,"disableStack":false,"fullStorage":false}], + jsonrpc: "2.0", + id: new Date().getTime() + }, function (error, result) { + console.log("piers!!!!!!!!! debug_traceTransaction sendResult|", result, "error", error) + console.log("piers failed|", result.failed); + console.log("piers structLogs|", result.structLogs); + //assert.equal(true, result === result); + assert.equal(undefined, result.failed) + }); + //console.log("sendResult3", sendResult3); + + //const doSomething = async () => { + // await sleep2(2000) + // //do stuff + //}; + //console.log("after sleep") + + const sendResult = await web3js.currentProvider.send({ + method: "eth_blockNumber", + params: [], + jsonrpc: "2.0", + id: new Date().getTime() + }, function (error, result) { + console.log("send eth_blockNumber|", result, "error", error) + }); + //console.log("sendResult", sendResult); + + + const receipt = await web3js.eth.getTransactionReceipt(txResult.transactionHash); + //const receipt = await web3js.eth.getTransactionReceipt(txResult.tx); + console.log("piers receipt|", receipt); + + //sleep(1000000) + + + + } catch(err) { + console.log("caught error", err); + } + //const dttResult = await web3js.currentProvider.send('debug_traceTransaction', [ txResult.transactionHash ]).then((result) => { + // console.log(result); + //}); + //console.log("dttResult", dttResult) + }) + +}); + +function sleep1(milliseconds) { + let timeStart = new Date().getTime(); + while (true) { + let elapsedTime = new Date().getTime() - timeStart; + if (elapsedTime > milliseconds) { + break; + } + } +} + +const sleep = (milliseconds) => { + return new Promise(resolve => setTimeout(resolve, milliseconds)) +} \ No newline at end of file diff --git a/rpc/debug/jsonrpc_conversion.go b/rpc/debug/jsonrpc_conversion.go index ad92ed00b7..050fd896c2 100644 --- a/rpc/debug/jsonrpc_conversion.go +++ b/rpc/debug/jsonrpc_conversion.go @@ -19,8 +19,16 @@ type JsonLogConfig struct { DisableStack bool `json:"disableStack,omitempty"` } -func DecTraceConfig(jcfg JsonTraceConfig) eth.TraceConfig { +func DecTraceConfig(jcfg *JsonTraceConfig) eth.TraceConfig { var logConfig *vm.LogConfig + if jcfg == nil { + return eth.TraceConfig{ + LogConfig: logConfig, + Tracer: nil, + Timeout: nil, + Reexec: nil, + } + } if jcfg.LogConfig != nil { logConfig = &vm.LogConfig{ DisableMemory: jcfg.LogConfig.DisableMemory, diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 87d4d85f7f..1010146865 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -35,6 +35,7 @@ func TraceTransaction( switch tracer := tracer.(type) { case *vm.StructLogger: return ðapi.ExecutionResult{ + Gas: 5, Failed: err == nil, ReturnValue: fmt.Sprintf("%x", result), StructLogs: ethapi.FormatLogs(tracer.StructLogs()), diff --git a/rpc/query_server.go b/rpc/query_server.go index 868a56aec8..7f5b212ea1 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -7,9 +7,8 @@ import ( "strconv" "strings" - "github.com/gorilla/websocket" - "github.com/gogo/protobuf/proto" + "github.com/gorilla/websocket" sha3 "github.com/miguelmota/go-solidity-sha3" "github.com/phonkee/go-pubsub" "github.com/pkg/errors" @@ -1140,7 +1139,7 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTra if err != nil { return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) } - cfg := debug.DecTraceConfig(*config) + cfg := debug.DecTraceConfig(config) memApp, err := s.InMemoryApp(blockNumber, s.BlockStore) if err != nil { return nil, err diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index 4aa44d1dd4..048505cd20 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -360,5 +360,10 @@ func (m *MultiWriterAppStore) VersionExists(version int64) bool { } func (m *MultiWriterAppStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - return m.appStore.RetrieveVersion(version) + reader, err := newMultiWriterVersionReader(*m, version) + if err != nil { + return nil, err + } + splitStore := newSplitStore(reader, NewMemStore()) + return splitStore, nil } diff --git a/store/splitstore.go b/store/splitstore.go index 918f85f2f3..daf43a0618 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -20,34 +20,34 @@ func newSplitStore(full KVReader, empty VersionedKVStore) VersionedKVStore { } func (ss splitStore) Get(key []byte) []byte { - if ss.KVReader.Has(key) { + if ss.VersionedKVStore.Has(key) { return ss.KVReader.Get(key) } if ss.deleted[string(key)] { return nil } - return ss.VersionedKVStore.Get(key) + return ss.KVReader.Get(key) } func (ss splitStore) Range(prefix []byte) plugin.RangeData { - resultRange := ss.KVReader.Range(prefix) - diskRange := ss.VersionedKVStore.Range(prefix) - for _, re := range diskRange { + readerRange := ss.KVReader.Range(prefix) + updateRange := ss.VersionedKVStore.Range(prefix) + for _, re := range updateRange { if !ss.KVReader.Has(re.Key) && !ss.deleted[string(re.Key)] { - resultRange = append(resultRange, re) + readerRange = append(readerRange, re) } } - return resultRange + return readerRange } func (ss splitStore) Has(key []byte) bool { - if ss.KVReader.Has(key) { + if ss.VersionedKVStore.Has(key) { return true } if ss.deleted[string(key)] { return false } - return ss.VersionedKVStore.Has(key) + return ss.KVReader.Has(key) } func (ss splitStore) Set(key, value []byte) { @@ -67,7 +67,7 @@ func (ss splitStore) Version() int64 { return 0 } func (ss splitStore) SaveVersion() ([]byte, int64, error) { - return nil, 0, errors.New("not implemented") + return nil, 0, nil } func (ss splitStore) Prune() error { diff --git a/store/store_test.go b/store/store_test.go index a2c3841ef5..296f208b88 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -255,7 +255,7 @@ func (ts *StoreTestSuite) VerifyRange(s KVReader, prefixes [][]byte, entries []* } actual := s.Range(prefixes[0]) require.Len(actual, len(expected), ts.StoreName) - if ts.StoreName != "MemStore" { + if ts.StoreName != "MemStore" && ts.StoreName != "SplitStore" { for i := range expected { require.EqualValues(expected[i], actual[i], ts.StoreName) } @@ -279,7 +279,7 @@ func (ts *StoreTestSuite) VerifyRange(s KVReader, prefixes [][]byte, entries []* require.Len(actual, len(expected), ts.StoreName) // TODO: MemStore keys should be iterated in ascending order - if ts.StoreName != "MemStore" { + if ts.StoreName != "MemStore" && ts.StoreName != "SplitStore" { for i := range expected { require.EqualValues(expected[i], actual[i], ts.StoreName) } @@ -297,7 +297,7 @@ func (ts *StoreTestSuite) VerifyRange(s KVReader, prefixes [][]byte, entries []* } actual = s.Range(prefixes[2]) require.Len(actual, len(expected), ts.StoreName) - if ts.StoreName != "MemStore" { + if ts.StoreName != "MemStore" && ts.StoreName != "SplitStore" { for i := range expected { require.EqualValues(expected[i], actual[i], ts.StoreName) } @@ -438,6 +438,26 @@ func (ts *MemStoreTestSuite) SetupSuite() { ts.supportsSnapshots = false } +func TestSplitStoreTestSuite(t *testing.T) { + suite.Run(t, &SplitStoreTestSuite{}) +} + +type SplitStoreTestSuite struct { + KVReader + VersionedKVStore + StoreTestSuite +} + +// runs before each test in this suite +func (ts *SplitStoreTestSuite) SetupTest() { + ts.store = newSplitStore(NewMemStore(), NewMemStore()) +} + +func (ts *SplitStoreTestSuite) SetupSuite() { + ts.StoreName = "SplitStore" + ts.supportsSnapshots = false +} + // // PruningIAVLStore // diff --git a/store/version_reader.go b/store/version_reader.go index 7b077f9b16..b39ce37575 100644 --- a/store/version_reader.go +++ b/store/version_reader.go @@ -1,7 +1,10 @@ package store import ( + "bytes" + "github.com/loomnetwork/go-loom/plugin" + "github.com/loomnetwork/go-loom/util" "github.com/pkg/errors" ) @@ -42,3 +45,55 @@ func (s *iAVLVersionReader) Has(key []byte) bool { value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) return value != nil } + +type multiWriterVersionReader struct { + appStore KVReader + evmStore KVReader + version int64 +} + +func newMultiWriterVersionReader(store MultiWriterAppStore, version int64) (KVReader, error) { + if !store.appStore.tree.VersionExists(version) { + return nil, errors.Errorf("version %d missing", version) + } + appStore, err := newIAVLVersionReader(*store.appStore, version) + if err != nil { + return nil, err + } + evmStore := NewEvmStore(store.evmStore.evmDB, 100) + if err := evmStore.LoadVersion(version); err != nil { + return nil, err + } + return &multiWriterVersionReader{ + appStore: appStore, + evmStore: evmStore, + version: version, + }, nil +} + +func (m *multiWriterVersionReader) Get(key []byte) []byte { + if util.HasPrefix(key, vmPrefix) { + return m.evmStore.Get(key) + } + return m.appStore.Get(key) +} + +// Range returns a range of keys +func (m *multiWriterVersionReader) Range(prefix []byte) plugin.RangeData { + if len(prefix) == 0 { + panic(errors.New("Range over nil prefix not implemented")) + } + + if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { + return m.evmStore.Range(prefix) + } + return m.appStore.Range(prefix) +} + +// Has checks if a key exists. +func (m *multiWriterVersionReader) Has(key []byte) bool { + if util.HasPrefix(key, vmPrefix) { + return m.evmStore.Has(key) + } + return m.appStore.Has(key) +} From 1c598691857cef1c24ee90b8d03c3b93249a4683 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 21 Oct 2019 10:53:03 +0100 Subject: [PATCH 16/49] merge-wip --- app.go | 5 ++- rpc/debug/trace.go | 18 ++++----- rpc/query_server.go | 40 ++++++++++--------- throttle/contract_tx_limiter_middleware.go | 2 +- throttle/evm.go | 2 +- throttle/legacy_deployer_middleware.go | 3 -- throttle/noevm.go | 2 +- throttle/throttle.go | 14 +++---- txhandler/middleware/deployer-middleware.go | 3 -- .../middleware/deployer-middleware_test.go | 3 +- .../middleware}/karma-middleware.go | 18 ++++----- .../middleware}/karma-middleware_test.go | 2 +- .../middleware}/middleware_test_helper.go | 4 +- 13 files changed, 53 insertions(+), 63 deletions(-) rename {throttle => txhandler/middleware}/karma-middleware.go (92%) rename {throttle => txhandler/middleware}/karma-middleware_test.go (99%) rename {throttle => txhandler/middleware}/middleware_test_helper.go (96%) diff --git a/app.go b/app.go index bb3086b935..d7f5f3a25a 100644 --- a/app.go +++ b/app.go @@ -10,7 +10,7 @@ import ( "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" - "github.com/pkg/errors" + //"github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" abci "github.com/tendermint/tendermint/abci/types" ttypes "github.com/tendermint/tendermint/types" @@ -19,7 +19,6 @@ import ( "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/registry" "github.com/loomnetwork/loomchain/txhandler" - "github.com/loomnetwork/loomchain/txhandler/middleware" "github.com/loomnetwork/loomchain/log" appstate "github.com/loomnetwork/loomchain/state" @@ -608,6 +607,7 @@ func (a *Application) ReadOnlyState() appstate.State { ) } +/* func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStore) (middleware.InMemoryApp, error) { startVersion := int64(blockNumber) - 1 if startVersion < 0 { @@ -634,3 +634,4 @@ func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStor a.TxHandlerFactory, ), nil } +*/ diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 1010146865..f474e35d94 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -3,25 +3,23 @@ package debug import ( - "fmt" - - "github.com/ethereum/go-ethereum/core/vm" + //"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/ethapi" - "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/pkg/errors" + //"github.com/ethereum/go-ethereum/eth/ethapi" + //"github.com/ethereum/go-ethereum/eth/tracers" + //"github.com/pkg/errors" "github.com/loomnetwork/loomchain/store" - "github.com/loomnetwork/loomchain/txhandler/middleware" ) func TraceTransaction( - app middleware.InMemoryApp, + //app middleware.InMemoryApp, blockstore store.BlockStore, blockNumber, txIndex int64, config eth.TraceConfig, ) (interface{}, error) { - if err := app.RunUpTo(blockNumber, txIndex); err != nil { + return nil, nil + /*if err := app.RunUpTo(blockNumber, txIndex); err != nil { return nil, err } @@ -45,5 +43,5 @@ func TraceTransaction( default: return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) } - + */ } diff --git a/rpc/query_server.go b/rpc/query_server.go index a59a984c1a..f0121818cf 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -63,7 +63,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State - InMemoryApp(uint64, store.BlockStore) (middleware.InMemoryApp, error) + //InMemoryApp(uint64, store.BlockStore) (middleware.InMemoryApp, error) } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -1119,24 +1119,26 @@ func (s *QueryServer) EthAccounts() ([]eth.Data, error) { } func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTraceConfig) (interface{}, error) { - receipt, err := s.EthGetTransactionReceipt(hash) - if err != nil || receipt == nil { - return nil, errors.Wrap(err, "cant find transaction matching hash") - } - blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) - if err != nil { - return nil, errors.Wrapf(err, "cant parse block number %v", receipt.BlockNumber) - } - txIndex, err := eth.DecQuantityToUint(receipt.TransactionIndex) - if err != nil { - return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) - } - cfg := debug.DecTraceConfig(config) - memApp, err := s.InMemoryApp(blockNumber, s.BlockStore) - if err != nil { - return nil, err - } - return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), int64(txIndex), cfg) + return nil, nil + /* + receipt, err := s.EthGetTransactionReceipt(hash) + if err != nil || receipt == nil { + return nil, errors.Wrap(err, "cant find transaction matching hash") + } + blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) + if err != nil { + return nil, errors.Wrapf(err, "cant parse block number %v", receipt.BlockNumber) + } + txIndex, err := eth.DecQuantityToUint(receipt.TransactionIndex) + if err != nil { + return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) + } + cfg := debug.DecTraceConfig(config) + memApp, err := s.InMemoryApp(blockNumber, s.BlockStore) + if err != nil { + return nil, err + } + return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), int64(txIndex), cfg)*/ } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { diff --git a/throttle/contract_tx_limiter_middleware.go b/throttle/contract_tx_limiter_middleware.go index b90c1f1484..95fbea7043 100644 --- a/throttle/contract_tx_limiter_middleware.go +++ b/throttle/contract_tx_limiter_middleware.go @@ -185,7 +185,7 @@ func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, if err := proto.Unmarshal(tx.Data, &msg); err != nil { return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) } - isDeploy, err := isEthDeploy(msg.Data) + isDeploy, err := IsEthDeploy(msg.Data) if err != nil { return res, err } diff --git a/throttle/evm.go b/throttle/evm.go index 7f2e73142e..444967ee7d 100644 --- a/throttle/evm.go +++ b/throttle/evm.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" ) -func isEthDeploy(txBytes []byte) (bool, error) { +func IsEthDeploy(txBytes []byte) (bool, error) { var tx types.Transaction if err := rlp.DecodeBytes(txBytes, &tx); err != nil { return false, errors.Wrap(err, "decoding ethereum transaction") diff --git a/throttle/legacy_deployer_middleware.go b/throttle/legacy_deployer_middleware.go index 6f524eb0ad..0855539349 100644 --- a/throttle/legacy_deployer_middleware.go +++ b/throttle/legacy_deployer_middleware.go @@ -9,9 +9,6 @@ import ( "github.com/pkg/errors" "github.com/loomnetwork/loomchain/auth/keys" - "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" diff --git a/throttle/noevm.go b/throttle/noevm.go index 1260d8894c..3ffd294bef 100644 --- a/throttle/noevm.go +++ b/throttle/noevm.go @@ -6,6 +6,6 @@ import ( "github.com/pkg/errors" ) -func isEthDeploy(_ []byte) (bool, error) { +func IsEthDeploy(_ []byte) (bool, error) { return false, errors.New("ethereum transactions not supported in non evm build") } diff --git a/throttle/throttle.go b/throttle/throttle.go index d9bb88f203..39bbe25ae5 100644 --- a/throttle/throttle.go +++ b/throttle/throttle.go @@ -27,7 +27,7 @@ const ( ) type Throttle struct { - maxCallCount int64 + MaxCallCount int64 sessionDuration int64 callLimiterPool map[string]*limiter.Limiter deployLimiterPool map[string]*limiter.Limiter @@ -44,7 +44,7 @@ func NewThrottle( maxCallCount int64, ) *Throttle { return &Throttle{ - maxCallCount: maxCallCount, + MaxCallCount: maxCallCount, sessionDuration: sessionDuration, callLimiterPool: make(map[string]*limiter.Limiter), deployLimiterPool: make(map[string]*limiter.Limiter), @@ -52,7 +52,7 @@ func NewThrottle( } } -func (t *Throttle) getNewLimiter(ctx context.Context, limit int64) *limiter.Limiter { +func (t *Throttle) GetNewLimiter(ctx context.Context, limit int64) *limiter.Limiter { rate := limiter.Rate{ Period: time.Duration(t.sessionDuration) * time.Second, Limit: limit, @@ -65,11 +65,11 @@ func (t *Throttle) getLimiterFromPool(ctx context.Context, limit int64) *limiter address := keys.Origin(ctx).String() _, ok := t.callLimiterPool[address] if !ok { - t.callLimiterPool[address] = t.getNewLimiter(ctx, limit) + t.callLimiterPool[address] = t.GetNewLimiter(ctx, limit) } if t.callLimiterPool[address].Rate.Limit != limit { delete(t.callLimiterPool, address) - t.callLimiterPool[address] = t.getNewLimiter(ctx, limit) + t.callLimiterPool[address] = t.GetNewLimiter(ctx, limit) } return t.callLimiterPool[address] @@ -91,7 +91,7 @@ func (t *Throttle) getLimiterContext( } } -func (t *Throttle) runThrottle( +func (t *Throttle) RunThrottle( state appstate.State, nonce uint64, origin loom.Address, limit int64, txId uint32, key string, ) error { limitCtx, err := t.getLimiterContext(state.Context(), nonce, limit, txId, key) @@ -112,7 +112,7 @@ func (t *Throttle) runThrottle( return nil } -func (t *Throttle) getKarmaForTransaction( +func (t *Throttle) GetKarmaForTransaction( karmaContractCtx contractpb.Context, origin loom.Address, isDeployTx bool, ) (*common.BigUInt, error) { // TODO: maybe should only count karma from active sources diff --git a/txhandler/middleware/deployer-middleware.go b/txhandler/middleware/deployer-middleware.go index 4569f924a8..d0918ec437 100644 --- a/txhandler/middleware/deployer-middleware.go +++ b/txhandler/middleware/deployer-middleware.go @@ -9,9 +9,6 @@ import ( "github.com/pkg/errors" "github.com/loomnetwork/loomchain/auth/keys" - "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" dw "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" "github.com/loomnetwork/loomchain/eth/utils" diff --git a/txhandler/middleware/deployer-middleware_test.go b/txhandler/middleware/deployer-middleware_test.go index d040f72ad2..409986062e 100644 --- a/txhandler/middleware/deployer-middleware_test.go +++ b/txhandler/middleware/deployer-middleware_test.go @@ -13,9 +13,8 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/go-loom/types" - loomAuth "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" dw "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" diff --git a/throttle/karma-middleware.go b/txhandler/middleware/karma-middleware.go similarity index 92% rename from throttle/karma-middleware.go rename to txhandler/middleware/karma-middleware.go index f04349bea3..ad7880a9b5 100644 --- a/throttle/karma-middleware.go +++ b/txhandler/middleware/karma-middleware.go @@ -1,4 +1,4 @@ -package throttle +package middleware import ( "fmt" @@ -13,12 +13,10 @@ import ( "github.com/pkg/errors" "github.com/loomnetwork/loomchain/auth/keys" - "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/builtin/plugins/karma" "github.com/loomnetwork/loomchain/eth/utils" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/throttle" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" ) @@ -31,7 +29,7 @@ func GetKarmaMiddleWare( sessionDuration int64, createKarmaContractCtx func(state appstate.State) (contractpb.Context, error), ) txhandler.TxMiddlewareFunc { - th := NewThrottle(sessionDuration, maxCallCount) + th := throttle.NewThrottle(sessionDuration, maxCallCount) return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, @@ -89,7 +87,7 @@ func GetKarmaMiddleWare( isDeployTx = true case types.TxID_ETHEREUM: - isDeployTx, err = isEthDeploy(msg.Data) + isDeployTx, err = throttle.IsEthDeploy(msg.Data) if err != nil { return res, err } @@ -129,7 +127,7 @@ func GetKarmaMiddleWare( return r, nil } - originKarma, err := th.getKarmaForTransaction(ctx, origin, isDeployTx) + originKarma, err := th.GetKarmaForTransaction(ctx, origin, isDeployTx) if err != nil { return res, errors.Wrap(err, "getting total karma") } @@ -160,11 +158,11 @@ func GetKarmaMiddleWare( if maxCallCount <= 0 { return res, errors.Errorf("max call count %d non positive", maxCallCount) } - callCount := th.maxCallCount + originKarmaTotal - if originKarmaTotal > math.MaxInt64-th.maxCallCount { + callCount := th.MaxCallCount + originKarmaTotal + if originKarmaTotal > math.MaxInt64-th.MaxCallCount { callCount = math.MaxInt64 } - err := th.runThrottle(state, nonceTx.Sequence, origin, callCount, tx.Id, karmaMiddlewareThrottleKey) + err := th.RunThrottle(state, nonceTx.Sequence, origin, callCount, tx.Id, karmaMiddlewareThrottleKey) if err != nil { return res, errors.Wrap(err, "call karma throttle") } diff --git a/throttle/karma-middleware_test.go b/txhandler/middleware/karma-middleware_test.go similarity index 99% rename from throttle/karma-middleware_test.go rename to txhandler/middleware/karma-middleware_test.go index e850cf1d19..c912b87191 100644 --- a/throttle/karma-middleware_test.go +++ b/txhandler/middleware/karma-middleware_test.go @@ -1,6 +1,6 @@ // +build evm -package throttle +package middleware import ( "context" diff --git a/throttle/middleware_test_helper.go b/txhandler/middleware/middleware_test_helper.go similarity index 96% rename from throttle/middleware_test_helper.go rename to txhandler/middleware/middleware_test_helper.go index c3ee80cbfa..69ee274db7 100644 --- a/throttle/middleware_test_helper.go +++ b/txhandler/middleware/middleware_test_helper.go @@ -1,4 +1,4 @@ -package throttle +package middleware import ( "context" @@ -9,8 +9,6 @@ import ( "github.com/loomnetwork/go-loom/types" "github.com/pkg/errors" - "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" loomAuth "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/eth/utils" appstate "github.com/loomnetwork/loomchain/state" From fbcfa43b1b35c1d26d99d1bc574657b777b755f0 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 21 Oct 2019 11:06:52 +0100 Subject: [PATCH 17/49] merge-wip --- auth/multi_chain_sigtx_middleware.go | 18 +++++------------- throttle/contract_tx_limiter_middleware.go | 11 ++++++----- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/auth/multi_chain_sigtx_middleware.go b/auth/multi_chain_sigtx_middleware.go index 1eaab5fa18..679e44d99a 100644 --- a/auth/multi_chain_sigtx_middleware.go +++ b/auth/multi_chain_sigtx_middleware.go @@ -36,14 +36,6 @@ const ( // Recovers the signer address from a signed tx. type originRecoveryFunc func(chainID string, tx SignedTx, allowedSigTypes []evmcompat.SignatureType) ([]byte, error) -var originRecoveryFuncs = map[keys.SignedTxType]originRecoveryFunc{ - keys.LoomSignedTxType: verifyEd25519, - keys.EthereumSignedTxType: verifySolidity66Byte, - keys.TronSignedTxType: verifyTron, - keys.BinanceSignedTxType: verifyBinance, -} - -type originRecoveryFunc func(tx SignedTx, allowedSigTypes []evmcompat.SignatureType) ([]byte, error) // NewMultiChainSignatureTxMiddleware returns tx signing middleware that supports a set of chain // specific signing algos. @@ -148,18 +140,18 @@ func NewMultiChainSignatureTxMiddleware( }) } -func getOriginRecoveryFunc(state appstate.State, txID types.TxID, txType SignedTxType) originRecoveryFunc { +func getOriginRecoveryFunc(state appstate.State, txID types.TxID, txType keys.SignedTxType) originRecoveryFunc { switch txType { - case LoomSignedTxType: + case keys.LoomSignedTxType: return verifyEd25519 - case EthereumSignedTxType: + case keys.EthereumSignedTxType: if (txID == types.TxID_ETHEREUM) && state.FeatureEnabled(features.EthTxFeature, false) { return VerifyWrappedEthTx } return verifySolidity66Byte - case TronSignedTxType: + case keys.TronSignedTxType: return verifyTron - case BinanceSignedTxType: + case keys.BinanceSignedTxType: return verifyBinance } return nil diff --git a/throttle/contract_tx_limiter_middleware.go b/throttle/contract_tx_limiter_middleware.go index 95fbea7043..72e490726e 100644 --- a/throttle/contract_tx_limiter_middleware.go +++ b/throttle/contract_tx_limiter_middleware.go @@ -15,6 +15,7 @@ import ( "github.com/loomnetwork/loomchain/auth" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" @@ -145,16 +146,16 @@ func loadTierMap(ctx contractpb.StaticContext) (map[udwtypes.TierID]udwtypes.Tie // sent to an EVM contract within a pre-configured block range. func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, createUserDeployerWhitelistCtx func(state appstate.State) (contractpb.Context, error), -) loomchain.TxMiddlewareFunc { +) txhandler.TxMiddlewareFunc { txl := &contractTxLimiter{ contractStatsMap: make(map[string]*contractStats), } - return loomchain.TxMiddlewareFunc(func( + return txhandler.TxMiddlewareFunc(func( state appstate.State, txBytes []byte, - next loomchain.TxHandlerFunc, + next txhandler.TxHandlerFunc, isCheckTx bool, - ) (res loomchain.TxHandlerResult, err error) { + ) (res txhandler.TxHandlerResult, err error) { if !isCheckTx { return next(state, txBytes, isCheckTx) } @@ -251,7 +252,7 @@ func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, } if txl.isAccountLimitReached(contractAddr, state.Block().Height) { - return loomchain.TxHandlerResult{}, ErrTxLimitReached + return txhandler.TxHandlerResult{}, ErrTxLimitReached } txl.updateState(contractAddr, state.Block().Height) From 7489a31d749e09d577700111056dc642db925884 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 21 Oct 2019 12:07:17 +0100 Subject: [PATCH 18/49] merge-wip --- rpc/query_server.go | 3 +-- throttle/contract_tx_limiter_middleware.go | 26 +++---------------- txhandler/factory/handler_factory.go | 2 +- txhandler/middleware/deployer-middleware.go | 5 ++-- .../middleware/middleware_test_helper.go | 3 ++- 5 files changed, 10 insertions(+), 29 deletions(-) diff --git a/rpc/query_server.go b/rpc/query_server.go index f0121818cf..4df93247c1 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -45,7 +45,6 @@ import ( "github.com/loomnetwork/loomchain/store" blockindex "github.com/loomnetwork/loomchain/store/block_index" evmaux "github.com/loomnetwork/loomchain/store/evm_aux" - "github.com/loomnetwork/loomchain/txhandler/middleware" lvm "github.com/loomnetwork/loomchain/vm" ) @@ -272,7 +271,7 @@ func (s *QueryServer) queryEvm(state appstate.State, caller, contract loom.Addre return nil, err } } - vm := levm.NewLoomVm(state, nil, nil, createABM, false, nil) + vm := levm.NewLoomVm(state, nil, createABM, false, nil) return vm.StaticCall(callerAddr, contract, query) } diff --git a/throttle/contract_tx_limiter_middleware.go b/throttle/contract_tx_limiter_middleware.go index 72e490726e..7b696d1336 100644 --- a/throttle/contract_tx_limiter_middleware.go +++ b/throttle/contract_tx_limiter_middleware.go @@ -5,20 +5,18 @@ import ( "time" "github.com/go-kit/kit/metrics" - kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" udwtypes "github.com/loomnetwork/go-loom/builtin/types/user_deployer_whitelist" "github.com/loomnetwork/go-loom/plugin/contractpb" ltypes "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain" + "github.com/pkg/errors" + "github.com/loomnetwork/loomchain/auth" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/pkg/errors" - stdprometheus "github.com/prometheus/client_golang/prometheus" ) var ( @@ -32,24 +30,6 @@ var ( contractTierMapLoadLatency metrics.Histogram ) -func init() { - fieldKeys := []string{"method", "error"} - tierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "loomchain", - Subsystem: "contract_tx_limiter_middleware", - Name: "tier_map_load_latency", - Help: "Total time taken for Tier Map to Load in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, fieldKeys) - contractTierMapLoadLatency = kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{ - Namespace: "loomchain", - Subsystem: "contract_tx_limiter_middleware", - Name: "contract_tier_map_load_latency", - Help: "Total time taken for Contract Tier Map to Load in seconds.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, fieldKeys) -} - type ContractTxLimiterConfig struct { // Enables the middleware Enabled bool @@ -163,7 +143,7 @@ func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { return res, errors.Wrap(err, "throttle: unwrap nonce Tx") } - var tx loomchain.Transaction + var tx ltypes.Transaction if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { return res, errors.New("throttle: unmarshal tx") } diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 153c0ba646..75f5bea5ca 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -97,7 +97,7 @@ func txMiddleWare( )) if cfg.Karma.Enabled { - txMiddleWare = append(txMiddleWare, throttle.GetKarmaMiddleWare( + txMiddleWare = append(txMiddleWare, middleware.GetKarmaMiddleWare( cfg.Karma.Enabled, cfg.Karma.MaxCallCount, cfg.Karma.SessionDuration, diff --git a/txhandler/middleware/deployer-middleware.go b/txhandler/middleware/deployer-middleware.go index d0918ec437..64b9c69c65 100644 --- a/txhandler/middleware/deployer-middleware.go +++ b/txhandler/middleware/deployer-middleware.go @@ -14,6 +14,7 @@ import ( "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/throttle" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" ) @@ -152,14 +153,14 @@ func NewDeployerWhitelistMiddleware( if err := proto.Unmarshal(tx.Data, &msg); err != nil { return res, errors.Wrap(err, "failed to unmarshal MessageTx") } - isDeploy, err := isEthDeploy(msg.Data) + isDeploy, err := throttle.IsEthDeploy(msg.Data) if err != nil { return res, err } if !isDeploy { break } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) ctx, err := createDeployerWhitelistCtx(state) if err != nil { return res, err diff --git a/txhandler/middleware/middleware_test_helper.go b/txhandler/middleware/middleware_test_helper.go index 69ee274db7..f15a7646e4 100644 --- a/txhandler/middleware/middleware_test_helper.go +++ b/txhandler/middleware/middleware_test_helper.go @@ -12,6 +12,7 @@ import ( loomAuth "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/eth/utils" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/throttle" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" ) @@ -70,7 +71,7 @@ func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.St }) case types.TxID_ETHEREUM: - isDeploy, err := isEthDeploy(msg.Data) + isDeploy, err := throttle.IsEthDeploy(msg.Data) if err != nil { return res, err } From cf0ced24ee0c115ee825dba8b237b4c90b629953 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 21 Oct 2019 15:13:13 +0100 Subject: [PATCH 19/49] Add version paramter to GetSnapshot --- app.go | 2 +- rpc/debug/trace.go | 3 ++- store/iavlstore.go | 2 +- store/logstore.go | 4 ++-- store/memstore.go | 2 +- store/multi_writer_app_store.go | 2 +- store/multi_writer_app_store_test.go | 16 ++++++++-------- store/pruning_iavlstore.go | 2 +- store/splitstore.go | 2 +- store/store.go | 2 +- store/store_test.go | 6 +++--- store/versioned_cachingstore.go | 4 ++-- store/versioned_cachingstore_test.go | 8 ++++---- 13 files changed, 28 insertions(+), 27 deletions(-) diff --git a/app.go b/app.go index d7f5f3a25a..88be251359 100644 --- a/app.go +++ b/app.go @@ -600,7 +600,7 @@ func (a *Application) ReadOnlyState() appstate.State { // not match the state... need to figure out why this hasn't spectacularly failed already return appstate.NewStoreStateSnapshot( nil, - a.Store.GetSnapshot(), + a.Store.GetSnapshot(0), a.lastBlockHeader, nil, // TODO: last block hash! a.GetValidatorSet, diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index f474e35d94..d547f70eaa 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -8,12 +8,13 @@ import ( //"github.com/ethereum/go-ethereum/eth/ethapi" //"github.com/ethereum/go-ethereum/eth/tracers" //"github.com/pkg/errors" - "github.com/loomnetwork/loomchain/store" + abci "github.com/tendermint/tendermint/abci/types" ) func TraceTransaction( //app middleware.InMemoryApp, + app abci.Application, blockstore store.BlockStore, blockNumber, txIndex int64, config eth.TraceConfig, diff --git a/store/iavlstore.go b/store/iavlstore.go index 8fd299e075..3fb8ea1521 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -209,7 +209,7 @@ func (s *IAVLStore) Prune() error { return nil } -func (s *IAVLStore) GetSnapshot() Snapshot { +func (s *IAVLStore) GetSnapshot(version int64) Snapshot { // This isn't an actual snapshot obviously, and never will be, but lets pretend... return &iavlStoreSnapshot{ IAVLStore: s, diff --git a/store/logstore.go b/store/logstore.go index 841437d185..7eee86bbaf 100644 --- a/store/logstore.go +++ b/store/logstore.go @@ -126,8 +126,8 @@ func (s *LogStore) Prune() error { return s.store.Prune() } -func (s *LogStore) GetSnapshot() Snapshot { - return s.store.GetSnapshot() +func (s *LogStore) GetSnapshot(version int64) Snapshot { + return s.store.GetSnapshot(version) } func (s *LogStore) VersionExists(version int64) bool { diff --git a/store/memstore.go b/store/memstore.go index 99c8984178..22c4913067 100644 --- a/store/memstore.go +++ b/store/memstore.go @@ -76,7 +76,7 @@ func (m *MemStore) Prune() error { return nil } -func (m *MemStore) GetSnapshot() Snapshot { +func (m *MemStore) GetSnapshot(version int64) Snapshot { panic("not implemented") } diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index 048505cd20..cc295ffb87 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -256,7 +256,7 @@ func (s *MultiWriterAppStore) Prune() error { return s.appStore.Prune() } -func (s *MultiWriterAppStore) GetSnapshot() Snapshot { +func (s *MultiWriterAppStore) GetSnapshot(version int64) Snapshot { defer func(begin time.Time) { getSnapshotDuration.Observe(time.Since(begin).Seconds()) }(time.Now()) diff --git a/store/multi_writer_app_store_test.go b/store/multi_writer_app_store_test.go index f08d244716..ce311ca558 100644 --- a/store/multi_writer_app_store_test.go +++ b/store/multi_writer_app_store_test.go @@ -112,7 +112,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShot() { store.Set(vmPrefixKey("abcde"), []byte("vvvvvvvvv")) store.Set([]byte("abcd"), []byte("asdfasdf")) - snapshot := store.GetSnapshot() + snapshot := store.GetSnapshot(0) require.Equal([]byte("hello"), snapshot.Get(vmPrefixKey("abcd"))) require.Equal([]byte("NewData"), snapshot.Get([]byte("abcd"))) require.Equal([]byte("world"), snapshot.Get(vmPrefixKey("abcde"))) @@ -120,7 +120,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShot() { _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot() + snapshot = store.GetSnapshot(0) require.Equal([]byte("asdfasdf"), snapshot.Get([]byte("abcd"))) require.Equal([]byte("hellooooooo"), snapshot.Get(vmPrefixKey("abcd"))) require.Equal([]byte("vvvvvvvvv"), snapshot.Get(vmPrefixKey("abcde"))) @@ -143,7 +143,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotFlushInter store.Set([]byte("test2"), []byte("test2v2")) // this snapshot is from memory - snapshotv1 := store.GetSnapshot() + snapshotv1 := store.GetSnapshot(0) require.Equal([]byte("test1"), snapshotv1.Get([]byte("test1"))) require.Equal([]byte("test2"), snapshotv1.Get([]byte("test2"))) @@ -152,7 +152,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotFlushInter require.NoError(err) // get snapshotv2 - snapshotv2 := store.GetSnapshot() + snapshotv2 := store.GetSnapshot(0) require.Equal([]byte("test1v2"), snapshotv2.Get([]byte("test1"))) require.Equal([]byte("test2v2"), snapshotv2.Get([]byte("test2"))) @@ -176,13 +176,13 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { store.Set([]byte("uuuu"), []byte("SSSSSSSSSSSSS")) store.Set([]byte("sssss"), []byte("NewData")) - snapshot := store.GetSnapshot() + snapshot := store.GetSnapshot(0) rangeData := snapshot.Range(vmPrefix) require.Equal(0, len(rangeData)) _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot() + snapshot = store.GetSnapshot(0) rangeData = snapshot.Range(vmPrefix) require.Equal(4+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, bytes.Compare(snapshot.Get(vmPrefixKey("abcd")), []byte("hello"))) @@ -194,7 +194,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { store.Delete(vmPrefixKey("abcd")) store.Delete([]byte("ssssvvv")) - snapshot = store.GetSnapshot() + snapshot = store.GetSnapshot(0) rangeData = snapshot.Range(vmPrefix) require.Equal(4+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, bytes.Compare(snapshot.Get(vmPrefixKey("abcd")), []byte("hello"))) @@ -205,7 +205,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot() + snapshot = store.GetSnapshot(0) rangeData = snapshot.Range(vmPrefix) require.Equal(3+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, len(snapshot.Get(vmPrefixKey("abcd")))) // has been deleted diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 65a6ab8634..dc198a1d48 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -177,7 +177,7 @@ func (s *PruningIAVLStore) Prune() error { return nil } -func (s *PruningIAVLStore) GetSnapshot() Snapshot { +func (s *PruningIAVLStore) GetSnapshot(version int64) Snapshot { // This isn't an actual snapshot obviously, and never will be, but lets pretend... return &pruningIAVLStoreSnapshot{ PruningIAVLStore: s, diff --git a/store/splitstore.go b/store/splitstore.go index daf43a0618..ab228ced8a 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -73,6 +73,6 @@ func (ss splitStore) SaveVersion() ([]byte, int64, error) { func (ss splitStore) Prune() error { return errors.New("not implemented") } -func (ss splitStore) GetSnapshot() Snapshot { +func (ss splitStore) GetSnapshot(version int64) Snapshot { return nil } diff --git a/store/store.go b/store/store.go index 1c5eb65d0c..3896be5696 100644 --- a/store/store.go +++ b/store/store.go @@ -53,7 +53,7 @@ type VersionedKVStore interface { SaveVersion() ([]byte, int64, error) // Delete old version of the store Prune() error - GetSnapshot() Snapshot + GetSnapshot(version int64) Snapshot VersionExists(int64) bool RetrieveVersion(int64) (VersionedKVStore, error) } diff --git a/store/store_test.go b/store/store_test.go index 296f208b88..1aa2ddcf44 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -348,7 +348,7 @@ func (ts *StoreTestSuite) VerifyConcurrentSnapshots() { if snap != nil { snap.Release() } - snap = ts.store.GetSnapshot() + snap = ts.store.GetSnapshot(0) } snap.Get([]byte(fmt.Sprintf("key/%d", i))) } @@ -395,7 +395,7 @@ func (ts *IAVLStoreTestSuite) TestSnapshotRange() { // but this store doesn't have real snapshots so the snapshot is expected to contain the same // unsaved state as the store itself... func() { - snap := ts.store.GetSnapshot() + snap := ts.store.GetSnapshot(0) defer snap.Release() ts.VerifyRange(snap, prefixes, entries) @@ -405,7 +405,7 @@ func (ts *IAVLStoreTestSuite) TestSnapshotRange() { // snapshot should see all the data that was saved to disk func() { - snap := ts.store.GetSnapshot() + snap := ts.store.GetSnapshot(0) defer snap.Release() ts.VerifyRange(snap, prefixes, entries) diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 2c1ab5c2ea..76f66e5774 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -371,9 +371,9 @@ func (c *versionedCachingStore) SaveVersion() ([]byte, int64, error) { return hash, version, err } -func (c *versionedCachingStore) GetSnapshot() Snapshot { +func (c *versionedCachingStore) GetSnapshot(version int64) Snapshot { return newVersionedCachingStoreSnapshot( - c.VersionedKVStore.GetSnapshot(), + c.VersionedKVStore.GetSnapshot(version), c.cache, c.version-1, c.logger, ) } diff --git a/store/versioned_cachingstore_test.go b/store/versioned_cachingstore_test.go index e0f7964c8d..c6c1836ffa 100644 --- a/store/versioned_cachingstore_test.go +++ b/store/versioned_cachingstore_test.go @@ -66,7 +66,7 @@ func (m *MockStore) RetrieveVersion(version int64) (VersionedKVStore, error) { panic("not implemented") } -func (m *MockStore) GetSnapshot() Snapshot { +func (m *MockStore) GetSnapshot(version int64) Snapshot { snapshotStore := make(map[string][]byte) for k, v := range m.storage { snapshotStore[k] = v @@ -106,7 +106,7 @@ func TestCachingStoreVersion(t *testing.T) { mockStore.Set(key2, []byte("value2")) mockStore.Set(key3, []byte("value3")) - snapshotv0 := cachingStore.GetSnapshot() + snapshotv0 := cachingStore.GetSnapshot(0) // cachingStoreSnapshot will cache key1 in memory as version 0 cachedValue := snapshotv0.Get(key1) @@ -122,7 +122,7 @@ func TestCachingStoreVersion(t *testing.T) { // save data into version 1 cachingStore.Set(key2, []byte("newvalue2")) cachingStore.Set(key3, []byte("newvalue3")) - snapshotv1 := cachingStore.GetSnapshot() + snapshotv1 := cachingStore.GetSnapshot(0) cachedValue = snapshotv1.Get(key2) assert.Equal(t, "newvalue2", string(cachedValue), "snapshotv1 should get correct value") cachedValue = snapshotv1.Get(key1) @@ -144,7 +144,7 @@ func TestCachingStoreVersion(t *testing.T) { // save to bump up version _, version, _ = cachingStore.SaveVersion() assert.Equal(t, int64(2), version, "version must be updated to 2") - snapshotv2 := cachingStore.GetSnapshot() + snapshotv2 := cachingStore.GetSnapshot(0) cachedValue = snapshotv2.Get(key1) assert.Equal(t, "value1", string(cachedValue), "snapshotv2 should get the value from cache") cachedValue = snapshotv2.Get(key2) From 048040c0447b55158ea9905acf21e67d6568f350 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 22 Oct 2019 14:20:24 +0100 Subject: [PATCH 20/49] Modify stores --- app.go | 19 +-- replay/replayapp.go | 134 ++++++++++++++++ rpc/debug/trace.go | 69 +++++++-- rpc/query_server.go | 41 +++-- store/iavlstore.go | 27 ++-- store/logstore.go | 8 - store/memstore.go | 8 - store/multi_writer_app_store.go | 137 ++++++++--------- store/pruning_iavlstore.go | 8 - store/splitstore.go | 2 +- store/store.go | 2 - store/store_test.go | 2 +- store/version_reader.go | 69 +-------- store/versioned_cachingstore.go | 8 - store/versioned_cachingstore_test.go | 8 - .../ethtx}/eth_tx_handler.go | 12 +- .../ethtx}/eth_tx_handler_noevm.go | 2 +- txhandler/middleware/memory_app.go | 145 ------------------ 18 files changed, 312 insertions(+), 389 deletions(-) create mode 100644 replay/replayapp.go rename {tx_handler => txhandler/ethtx}/eth_tx_handler.go (92%) rename {tx_handler => txhandler/ethtx}/eth_tx_handler_noevm.go (96%) delete mode 100644 txhandler/middleware/memory_app.go diff --git a/app.go b/app.go index 88be251359..490d94cd59 100644 --- a/app.go +++ b/app.go @@ -10,6 +10,8 @@ import ( "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" + "github.com/pkg/errors" + //"github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" abci "github.com/tendermint/tendermint/abci/types" @@ -18,6 +20,7 @@ import ( "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/registry" + "github.com/loomnetwork/loomchain/replay" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/log" @@ -595,6 +598,7 @@ func (a *Application) Query(req abci.RequestQuery) abci.ResponseQuery { func (a *Application) height() int64 { return a.Store.Version() + 1 } + func (a *Application) ReadOnlyState() appstate.State { // TODO: the store snapshot should be created atomically, otherwise the block header might // not match the state... need to figure out why this hasn't spectacularly failed already @@ -607,23 +611,21 @@ func (a *Application) ReadOnlyState() appstate.State { ) } -/* -func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStore) (middleware.InMemoryApp, error) { +func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (replay.ReplayApplication, error) { startVersion := int64(blockNumber) - 1 if startVersion < 0 { return nil, errors.Errorf("invalid block number %d", blockNumber) } - for ; !a.Store.VersionExists(startVersion) || startVersion <= int64(0); startVersion-- { + var snapshot store.Snapshot + for ; snapshot != nil || startVersion > 0; snapshot = a.Store.GetSnapshot(startVersion) { + startVersion-- } if startVersion == 0 { return nil, errors.Errorf("no saved version for height %d", blockNumber) } - startStore, err := a.Store.RetrieveVersion(startVersion) - if err != nil { - return nil, err - } - return middleware.NewInMemoryApp( + startStore := store.NewSplitStore(snapshot, store.NewMemStore()) + return replay.NewReplayApplication( startVersion, blockstore, startStore, @@ -634,4 +636,3 @@ func (a *Application) InMemoryApp(blockNumber uint64, blockstore store.BlockStor a.TxHandlerFactory, ), nil } -*/ diff --git a/replay/replayapp.go b/replay/replayapp.go new file mode 100644 index 0000000000..9a59176e54 --- /dev/null +++ b/replay/replayapp.go @@ -0,0 +1,134 @@ +package replay + +import ( + "context" + + "github.com/ethereum/go-ethereum/core/vm" + "github.com/loomnetwork/go-loom/builtin/types/chainconfig" + "github.com/loomnetwork/go-loom/types" + abci "github.com/tendermint/tendermint/abci/types" + ttypes "github.com/tendermint/tendermint/types" + + "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" +) + +type ReplayApplication interface { + abci.Application + SetTracer(vm.Tracer) +} + +var _ abci.Application = (*replayApplication)(nil) +var _ abci.Application = ReplayApplication(nil) + +type replayApplication struct { + height int64 + blockstore store.BlockStore + store store.VersionedKVStore + txHandler txhandler.TxHandler + receiptsVersion int32 + getValidatorSet state.GetValidatorSet + config *chainconfig.Config + txHandlerFactory txhandler.TxHandlerFactory + tracer vm.Tracer +} + +func NewReplayApplication( + height int64, + blockstore store.BlockStore, + store store.VersionedKVStore, + txHandler txhandler.TxHandler, + receiptsVersion int32, + getValidatorSet state.GetValidatorSet, + config *chainconfig.Config, + txHandlerFactory txhandler.TxHandlerFactory, +) *replayApplication { + return &replayApplication{ + height: height, + blockstore: blockstore, + store: store, + txHandler: txHandler, + receiptsVersion: receiptsVersion, + getValidatorSet: getValidatorSet, + config: config, + txHandlerFactory: txHandlerFactory, + } +} + +func (a *replayApplication) SetTracer(tracer vm.Tracer) { + a.tracer = tracer +} + +func (a *replayApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { + splitStoreTx := store.WrapAtomic(a.store).BeginTx() + defer splitStoreTx.Rollback() + resultBlock, err := a.blockstore.GetBlockByHeight(&a.height) + if err != nil { + return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} + } + + storeState := state.NewStoreState2( + context.Background(), + splitStoreTx, + blockHeaderFromHeader(resultBlock.BlockMeta.Header), + a.getValidatorSet, + ).WithOnChainConfig(a.config) + txHandle, err := a.txHandlerFactory.TxHandler(a.tracer, false) + if err != nil { + return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} + } + + r, err := txHandle.ProcessTx(storeState, tx, false) + if err != nil { + return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} + } + return abci.ResponseDeliverTx{Code: abci.CodeTypeOK, Data: r.Data, Tags: r.Tags, Info: r.Info} +} + +func (_ *replayApplication) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginBlock { + return abci.ResponseBeginBlock{} +} + +func (a *replayApplication) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock { + a.height++ + return abci.ResponseEndBlock{} +} + +func (_ *replayApplication) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { + return abci.ResponseInitChain{} +} + +func (_ *replayApplication) Info(req abci.RequestInfo) abci.ResponseInfo { + return abci.ResponseInfo{} +} + +func (_ *replayApplication) SetOption(req abci.RequestSetOption) abci.ResponseSetOption { + return abci.ResponseSetOption{} +} + +func (_ *replayApplication) CheckTx(tx []byte) abci.ResponseCheckTx { + return abci.ResponseCheckTx{Code: 0} +} + +func (_ *replayApplication) Commit() abci.ResponseCommit { + return abci.ResponseCommit{} +} + +func (_ *replayApplication) Query(req abci.RequestQuery) abci.ResponseQuery { + return abci.ResponseQuery{Code: 0} +} + +func blockHeaderFromHeader(header ttypes.Header) types.BlockHeader { + return types.BlockHeader{ + ChainID: header.ChainID, + Height: header.Height, + Time: int64(header.Time.Unix()), + NumTxs: int32(header.NumTxs), + LastBlockID: types.BlockID{ + Hash: header.LastBlockID.Hash, + }, + ValidatorsHash: header.ValidatorsHash, + AppHash: header.AppHash, + } +} diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index d547f70eaa..929833fcab 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -3,39 +3,45 @@ package debug import ( - //"github.com/ethereum/go-ethereum/core/vm" + "fmt" + + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" - //"github.com/ethereum/go-ethereum/eth/ethapi" - //"github.com/ethereum/go-ethereum/eth/tracers" - //"github.com/pkg/errors" - "github.com/loomnetwork/loomchain/store" + "github.com/pkg/errors" abci "github.com/tendermint/tendermint/abci/types" + + "github.com/ethereum/go-ethereum/eth/ethapi" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/loomnetwork/loomchain/replay" + "github.com/loomnetwork/loomchain/store" ) func TraceTransaction( - //app middleware.InMemoryApp, - app abci.Application, + app replay.ReplayApplication, blockstore store.BlockStore, - blockNumber, txIndex int64, + startBlockNumber, targetBlockNumber, txIndex int64, config eth.TraceConfig, ) (interface{}, error) { - return nil, nil - /*if err := app.RunUpTo(blockNumber, txIndex); err != nil { + if err := runUpTo(app, blockstore, startBlockNumber, targetBlockNumber, txIndex); err != nil { return nil, err } - block, err := blockstore.GetBlockByHeight(&blockNumber) + block, err := blockstore.GetBlockByHeight(&targetBlockNumber) if err != nil { - return nil, errors.Wrapf(err, "getting block information at height %v", blockNumber) + return nil, errors.Wrapf(err, "getting block information at height %v", targetBlockNumber) } - - result, tracer, err := app.TraceProcessTx(block.Block.Data.Txs[txIndex], config) + tracer, err := createTracer(config) + if err != nil { + return nil, err + } + app.SetTracer(tracer) + result := app.DeliverTx(block.Block.Data.Txs[txIndex]) switch tracer := tracer.(type) { case *vm.StructLogger: return ðapi.ExecutionResult{ Gas: 5, - Failed: err == nil, + Failed: result.Code != abci.CodeTypeOK, ReturnValue: fmt.Sprintf("%x", result), StructLogs: ethapi.FormatLogs(tracer.StructLogs()), }, nil @@ -44,5 +50,36 @@ func TraceTransaction( default: return nil, errors.New(fmt.Sprintf("bad tracer type %T", tracer)) } - */ +} + +func runUpTo(app abci.Application, blockstore store.BlockStore, startHeight, height, index int64) error { + for h := startHeight; h <= height; h++ { + _ = app.BeginBlock(abci.RequestBeginBlock{}) + + block, err := blockstore.GetBlockByHeight(&h) + if err != nil { + return errors.Wrapf(err, "getting block information at height %v", h) + } + for i := 0; i < len(block.Block.Data.Txs); i++ { + if h != height || i != int(index) { + _ = app.DeliverTx(block.Block.Data.Txs[i]) + } else { + return nil + } + } + + _ = app.EndBlock(abci.RequestEndBlock{}) + } + return errors.Errorf("cannot find transaction at height %d index %d", height, index) +} + +func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { + if traceCfg.Tracer == nil || len(*traceCfg.Tracer) == 0 { + return vm.NewStructLogger(traceCfg.LogConfig), nil + } + tracer, err := tracers.New(*traceCfg.Tracer) + if err != nil { + return nil, err + } + return tracer, nil } diff --git a/rpc/query_server.go b/rpc/query_server.go index 4df93247c1..39ed4c13b0 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -39,6 +39,7 @@ import ( "github.com/loomnetwork/loomchain/receipts/common" "github.com/loomnetwork/loomchain/registry" registryFac "github.com/loomnetwork/loomchain/registry/factory" + "github.com/loomnetwork/loomchain/replay" "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" appstate "github.com/loomnetwork/loomchain/state" @@ -62,7 +63,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State - //InMemoryApp(uint64, store.BlockStore) (middleware.InMemoryApp, error) + ReplayApplication(uint64, store.BlockStore) (replay.ReplayApplication, error) } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -1118,26 +1119,24 @@ func (s *QueryServer) EthAccounts() ([]eth.Data, error) { } func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTraceConfig) (interface{}, error) { - return nil, nil - /* - receipt, err := s.EthGetTransactionReceipt(hash) - if err != nil || receipt == nil { - return nil, errors.Wrap(err, "cant find transaction matching hash") - } - blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) - if err != nil { - return nil, errors.Wrapf(err, "cant parse block number %v", receipt.BlockNumber) - } - txIndex, err := eth.DecQuantityToUint(receipt.TransactionIndex) - if err != nil { - return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) - } - cfg := debug.DecTraceConfig(config) - memApp, err := s.InMemoryApp(blockNumber, s.BlockStore) - if err != nil { - return nil, err - } - return debug.TraceTransaction(memApp, s.BlockStore, int64(blockNumber), int64(txIndex), cfg)*/ + receipt, err := s.EthGetTransactionReceipt(hash) + if err != nil || receipt == nil { + return nil, errors.Wrap(err, "cant find transaction matching hash") + } + blockNumber, err := eth.DecQuantityToUint(receipt.BlockNumber) + if err != nil { + return nil, errors.Wrapf(err, "cant parse block number %v", receipt.BlockNumber) + } + txIndex, err := eth.DecQuantityToUint(receipt.TransactionIndex) + if err != nil { + return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) + } + cfg := debug.DecTraceConfig(config) + replayApp, err := s.ReplayApplication(blockNumber, s.BlockStore) + if err != nil { + return nil, err + } + return debug.TraceTransaction(replayApp, s.BlockStore, 1, int64(blockNumber), int64(txIndex), cfg) } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { diff --git a/store/iavlstore.go b/store/iavlstore.go index 3fb8ea1521..e45507d0a8 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -210,23 +210,20 @@ func (s *IAVLStore) Prune() error { } func (s *IAVLStore) GetSnapshot(version int64) Snapshot { - // This isn't an actual snapshot obviously, and never will be, but lets pretend... - return &iavlStoreSnapshot{ - IAVLStore: s, + if version == 0 { + // This isn't an actual snapshot obviously, and never will be, but lets pretend... + return &iavlStoreSnapshot{ + IAVLStore: s, + } } -} - -func (s *IAVLStore) VersionExists(version int64) bool { - return s.tree.VersionExists(version) -} - -func (s *IAVLStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - reader, err := newIAVLVersionReader(*s, version) - if err != nil { - return nil, err + if s.tree.VersionExists(version) { + reader, err := newIAVLVersionReader(*s, version) + if err != nil { + return nil + } + return reader } - splitStore := newSplitStore(reader, NewMemStore()) - return splitStore, nil + return nil } // NewIAVLStore creates a new IAVLStore. diff --git a/store/logstore.go b/store/logstore.go index 7eee86bbaf..2bbcf72047 100644 --- a/store/logstore.go +++ b/store/logstore.go @@ -129,11 +129,3 @@ func (s *LogStore) Prune() error { func (s *LogStore) GetSnapshot(version int64) Snapshot { return s.store.GetSnapshot(version) } - -func (s *LogStore) VersionExists(version int64) bool { - return s.store.VersionExists(version) -} - -func (s *LogStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - return s.store.RetrieveVersion(version) -} diff --git a/store/memstore.go b/store/memstore.go index 22c4913067..162db180a8 100644 --- a/store/memstore.go +++ b/store/memstore.go @@ -79,11 +79,3 @@ func (m *MemStore) Prune() error { func (m *MemStore) GetSnapshot(version int64) Snapshot { panic("not implemented") } - -func (m *MemStore) VersionExists(version int64) bool { - panic("not implemented") -} - -func (m *MemStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - panic("not implemented") -} diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index cc295ffb87..f7740b20ee 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -11,12 +11,11 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/util" - "github.com/loomnetwork/loomchain/db" - "github.com/loomnetwork/loomchain/features" - "github.com/loomnetwork/loomchain/log" "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" "github.com/tendermint/iavl" + + "github.com/loomnetwork/loomchain/features" ) var ( @@ -119,7 +118,9 @@ func NewMultiWriterAppStore( store.onlySaveEvmStateToEvmStore = bytes.Equal(store.appStore.Get(evmDBFeatureKey), []byte{1}) } - store.setLastSavedTreeToVersion(appStore.Version()) + if err := store.setLastSavedTreeToVersion(appStore.Version()); err != nil { + return nil, err + } return store, nil } @@ -260,40 +261,42 @@ func (s *MultiWriterAppStore) GetSnapshot(version int64) Snapshot { defer func(begin time.Time) { getSnapshotDuration.Observe(time.Since(begin).Seconds()) }(time.Now()) - appStoreTree := (*iavl.ImmutableTree)(atomic.LoadPointer(&s.lastSavedTree)) - evmDbSnapshot := s.evmStore.GetSnapshot(appStoreTree.Version()) - return newMultiWriterStoreSnapshot(evmDbSnapshot, appStoreTree) + + if version == 0 { + version = (*iavl.ImmutableTree)(atomic.LoadPointer(&s.lastSavedTree)).Version() + } + return newMultiWriterStoreSnapshot(*s, version) } type multiWriterStoreSnapshot struct { - evmDbSnapshot db.Snapshot - appStoreTree *iavl.ImmutableTree + evmDbSnapshot KVReader + appStoreSnapshot Snapshot } -func newMultiWriterStoreSnapshot(evmDbSnapshot db.Snapshot, appStoreTree *iavl.ImmutableTree) *multiWriterStoreSnapshot { +func newMultiWriterStoreSnapshot(store MultiWriterAppStore, version int64) *multiWriterStoreSnapshot { + evmStore := NewEvmStore(store.evmStore.evmDB, 100) return &multiWriterStoreSnapshot{ - evmDbSnapshot: evmDbSnapshot, - appStoreTree: appStoreTree, + evmDbSnapshot: evmStore, + appStoreSnapshot: store.appStore.GetSnapshot(version), } } func (s *multiWriterStoreSnapshot) Release() { - s.evmDbSnapshot.Release() - s.appStoreTree = nil + s.appStoreSnapshot.Release() } func (s *multiWriterStoreSnapshot) Has(key []byte) bool { if util.HasPrefix(key, vmPrefix) { return s.evmDbSnapshot.Has(key) } - return s.appStoreTree.Has(key) + return s.appStoreSnapshot.Has(key) } func (s *multiWriterStoreSnapshot) Get(key []byte) []byte { if util.HasPrefix(key, vmPrefix) { return s.evmDbSnapshot.Get(key) } - _, val := s.appStoreTree.Get(key) + val := s.appStoreSnapshot.Get(key) return val } @@ -303,67 +306,61 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { panic(errors.New("Range over nil prefix not implemented")) } - ret := make(plugin.RangeData, 0) - if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { - it := s.evmDbSnapshot.NewIterator(prefix, prefixRangeEnd(prefix)) - defer it.Close() - - for ; it.Valid(); it.Next() { - key := it.Key() - if util.HasPrefix(key, prefix) { - var err error - key, err = util.UnprefixKey(key, prefix) - if err != nil { - panic(err) + return s.evmDbSnapshot.Range(prefix) + } + return s.appStoreSnapshot.Range(prefix) + /* + // Seems to repeat code in //func (s *EvmStore) Range(prefix []byte) plugin.RangeData ???? + ret := make(plugin.RangeData, 0) + + if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { + it := s.evmDbSnapshot.NewIterator(prefix, prefixRangeEnd(prefix)) + defer it.Close() + + for ; it.Valid(); it.Next() { + key := it.Key() + if util.HasPrefix(key, prefix) { + var err error + key, err = util.UnprefixKey(key, prefix) + if err != nil { + panic(err) + } + + ret = append(ret, &plugin.RangeEntry{ + Key: key, + Value: it.Value(), + }) } - - ret = append(ret, &plugin.RangeEntry{ - Key: key, - Value: it.Value(), - }) } + return ret } - return ret - } - - // Otherwise iterate over the IAVL tree - keys, values, _, err := s.appStoreTree.GetRangeWithProof(prefix, prefixRangeEnd(prefix), 0) - if err != nil { - log.Error("failed to get range", "prefix", string(prefix), "err", err) - return ret - } - for i, k := range keys { - // Tree range gives all keys that has prefix but it does not check zero byte - // after the prefix. So we have to check zero byte after prefix using util.HasPrefix - if util.HasPrefix(k, prefix) { - k, err = util.UnprefixKey(k, prefix) - if err != nil { - panic(err) - } - } else { // Skip this key as it does not have the prefix - continue + // Otherwise iterate over the IAVL tree + keys, values, _, err := s.appStoreTree.GetRangeWithProof(prefix, prefixRangeEnd(prefix), 0) + if err != nil { + log.Error("failed to get range", "prefix", string(prefix), "err", err) + return ret } - ret = append(ret, &plugin.RangeEntry{ - Key: k, - Value: values[i], - }) - } - - return ret -} + for i, k := range keys { + // Tree range gives all keys that has prefix but it does not check zero byte + // after the prefix. So we have to check zero byte after prefix using util.HasPrefix + if util.HasPrefix(k, prefix) { + k, err = util.UnprefixKey(k, prefix) + if err != nil { + panic(err) + } + } else { // Skip this key as it does not have the prefix + continue + } -func (m *MultiWriterAppStore) VersionExists(version int64) bool { - return m.appStore.VersionExists(version) -} + ret = append(ret, &plugin.RangeEntry{ + Key: k, + Value: values[i], + }) + } -func (m *MultiWriterAppStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - reader, err := newMultiWriterVersionReader(*m, version) - if err != nil { - return nil, err - } - splitStore := newSplitStore(reader, NewMemStore()) - return splitStore, nil + return ret + */ } diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index dc198a1d48..8b4f7b10a7 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -269,11 +269,3 @@ type pruningIAVLStoreSnapshot struct { func (s *pruningIAVLStoreSnapshot) Release() { // noop } - -func (m *PruningIAVLStore) VersionExists(version int64) bool { - return m.store.VersionExists(version) -} - -func (m *PruningIAVLStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - return m.store.RetrieveVersion(version) -} diff --git a/store/splitstore.go b/store/splitstore.go index ab228ced8a..aec422be30 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -11,7 +11,7 @@ type splitStore struct { deleted map[string]bool } -func newSplitStore(full KVReader, empty VersionedKVStore) VersionedKVStore { +func NewSplitStore(full KVReader, empty VersionedKVStore) VersionedKVStore { return &splitStore{ KVReader: full, VersionedKVStore: empty, diff --git a/store/store.go b/store/store.go index 3896be5696..167b19254b 100644 --- a/store/store.go +++ b/store/store.go @@ -54,8 +54,6 @@ type VersionedKVStore interface { // Delete old version of the store Prune() error GetSnapshot(version int64) Snapshot - VersionExists(int64) bool - RetrieveVersion(int64) (VersionedKVStore, error) } type cacheItem struct { diff --git a/store/store_test.go b/store/store_test.go index 1aa2ddcf44..941c0b5b0f 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -450,7 +450,7 @@ type SplitStoreTestSuite struct { // runs before each test in this suite func (ts *SplitStoreTestSuite) SetupTest() { - ts.store = newSplitStore(NewMemStore(), NewMemStore()) + ts.store = NewSplitStore(NewMemStore(), NewMemStore()) } func (ts *SplitStoreTestSuite) SetupSuite() { diff --git a/store/version_reader.go b/store/version_reader.go index b39ce37575..bd2d72debc 100644 --- a/store/version_reader.go +++ b/store/version_reader.go @@ -1,35 +1,31 @@ package store import ( - "bytes" - "github.com/loomnetwork/go-loom/plugin" - "github.com/loomnetwork/go-loom/util" "github.com/pkg/errors" ) -type iAVLVersionReader struct { +type iAVLSnapshot struct { IAVLStore version int64 } -func newIAVLVersionReader(store IAVLStore, version int64) (KVReader, error) { +func newIAVLVersionReader(store IAVLStore, version int64) (Snapshot, error) { if !store.tree.VersionExists(version) { return nil, errors.Errorf("version %d missing", version) } - return &iAVLVersionReader{ + return &iAVLSnapshot{ IAVLStore: store, version: version, }, nil } -func (s *iAVLVersionReader) Get(key []byte) []byte { +func (s *iAVLSnapshot) Get(key []byte) []byte { value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) return value } -// Range returns a range of keys -func (s *iAVLVersionReader) Range(prefix []byte) plugin.RangeData { +func (s *iAVLSnapshot) Range(prefix []byte) plugin.RangeData { rangeData := s.IAVLStore.Range(prefix) var rtv plugin.RangeData for _, entry := range rangeData { @@ -40,60 +36,9 @@ func (s *iAVLVersionReader) Range(prefix []byte) plugin.RangeData { return rtv } -// Has checks if a key exists. -func (s *iAVLVersionReader) Has(key []byte) bool { +func (s *iAVLSnapshot) Has(key []byte) bool { value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) return value != nil } -type multiWriterVersionReader struct { - appStore KVReader - evmStore KVReader - version int64 -} - -func newMultiWriterVersionReader(store MultiWriterAppStore, version int64) (KVReader, error) { - if !store.appStore.tree.VersionExists(version) { - return nil, errors.Errorf("version %d missing", version) - } - appStore, err := newIAVLVersionReader(*store.appStore, version) - if err != nil { - return nil, err - } - evmStore := NewEvmStore(store.evmStore.evmDB, 100) - if err := evmStore.LoadVersion(version); err != nil { - return nil, err - } - return &multiWriterVersionReader{ - appStore: appStore, - evmStore: evmStore, - version: version, - }, nil -} - -func (m *multiWriterVersionReader) Get(key []byte) []byte { - if util.HasPrefix(key, vmPrefix) { - return m.evmStore.Get(key) - } - return m.appStore.Get(key) -} - -// Range returns a range of keys -func (m *multiWriterVersionReader) Range(prefix []byte) plugin.RangeData { - if len(prefix) == 0 { - panic(errors.New("Range over nil prefix not implemented")) - } - - if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { - return m.evmStore.Range(prefix) - } - return m.appStore.Range(prefix) -} - -// Has checks if a key exists. -func (m *multiWriterVersionReader) Has(key []byte) bool { - if util.HasPrefix(key, vmPrefix) { - return m.evmStore.Has(key) - } - return m.appStore.Has(key) -} +func (s *iAVLSnapshot) Release() {} diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 76f66e5774..7f19baa266 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -378,14 +378,6 @@ func (c *versionedCachingStore) GetSnapshot(version int64) Snapshot { ) } -func (c *versionedCachingStore) VersionExists(version int64) bool { - return c.VersionedKVStore.VersionExists(version) -} - -func (c *versionedCachingStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - return c.VersionedKVStore.RetrieveVersion(version) -} - // CachingStoreSnapshot is a read-only CachingStore with specified version type versionedCachingStoreSnapshot struct { Snapshot diff --git a/store/versioned_cachingstore_test.go b/store/versioned_cachingstore_test.go index c6c1836ffa..a332d5bab0 100644 --- a/store/versioned_cachingstore_test.go +++ b/store/versioned_cachingstore_test.go @@ -58,14 +58,6 @@ func (m *MockStore) Prune() error { return nil } -func (m *MockStore) VersionExists(version int64) bool { - panic("not implemented") -} - -func (m *MockStore) RetrieveVersion(version int64) (VersionedKVStore, error) { - panic("not implemented") -} - func (m *MockStore) GetSnapshot(version int64) Snapshot { snapshotStore := make(map[string][]byte) for k, v := range m.storage { diff --git a/tx_handler/eth_tx_handler.go b/txhandler/ethtx/eth_tx_handler.go similarity index 92% rename from tx_handler/eth_tx_handler.go rename to txhandler/ethtx/eth_tx_handler.go index 8630374594..df0f47c7c2 100644 --- a/tx_handler/eth_tx_handler.go +++ b/txhandler/ethtx/eth_tx_handler.go @@ -1,6 +1,6 @@ // +build evm -package tx_handler +package ethtx import ( "fmt" @@ -12,12 +12,12 @@ import ( "github.com/loomnetwork/go-loom/types" "github.com/pkg/errors" - "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/registry/factory" appstate "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" ) @@ -31,8 +31,8 @@ func (h *EthTxHandler) ProcessTx( state appstate.State, txBytes []byte, isCheckTx bool, -) (loomchain.TxHandlerResult, error) { - var r loomchain.TxHandlerResult +) (txhandler.TxHandlerResult, error) { + var r txhandler.TxHandlerResult if !state.FeatureEnabled(features.EthTxFeature, false) { return r, errors.New("ethereum transactions feature not enabled") @@ -43,7 +43,7 @@ func (h *EthTxHandler) ProcessTx( return r, err } - origin := auth.Origin(state.Context()) + origin := keys.Origin(state.Context()) caller := loom.UnmarshalAddressPB(msg.From) if caller.Compare(origin) != 0 { diff --git a/tx_handler/eth_tx_handler_noevm.go b/txhandler/ethtx/eth_tx_handler_noevm.go similarity index 96% rename from tx_handler/eth_tx_handler_noevm.go rename to txhandler/ethtx/eth_tx_handler_noevm.go index 07404f5ac4..5012d4d984 100644 --- a/tx_handler/eth_tx_handler_noevm.go +++ b/txhandler/ethtx/eth_tx_handler_noevm.go @@ -1,6 +1,6 @@ // +build !evm -package tx_handler +package ethtx import ( "github.com/loomnetwork/loomchain" diff --git a/txhandler/middleware/memory_app.go b/txhandler/middleware/memory_app.go deleted file mode 100644 index cb0c140576..0000000000 --- a/txhandler/middleware/memory_app.go +++ /dev/null @@ -1,145 +0,0 @@ -package middleware - -import ( - "context" - - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/loomnetwork/go-loom/builtin/types/chainconfig" - "github.com/loomnetwork/go-loom/types" - "github.com/pkg/errors" - ttypes "github.com/tendermint/tendermint/types" - - "github.com/loomnetwork/loomchain/state" - "github.com/loomnetwork/loomchain/store" - "github.com/loomnetwork/loomchain/txhandler" -) - -type InMemoryApp interface { - ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) - TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) - RunUpTo(height, index int64) error - NextBlock() - Height() int64 -} - -type inMemoryApp struct { - height int64 - blockstore store.BlockStore - store store.VersionedKVStore - txHandler txhandler.TxHandler - receiptsVersion int32 - getValidatorSet state.GetValidatorSet - config *chainconfig.Config - txHandlerFactory txhandler.TxHandlerFactory -} - -func NewInMemoryApp( - height int64, - blockstore store.BlockStore, - store store.VersionedKVStore, - txHandler txhandler.TxHandler, - receiptsVersion int32, - getValidatorSet state.GetValidatorSet, - config *chainconfig.Config, - txHandlerFactory txhandler.TxHandlerFactory, -) InMemoryApp { - return &inMemoryApp{ - height: height, - blockstore: blockstore, - store: store, - txHandler: txHandler, - receiptsVersion: receiptsVersion, - getValidatorSet: getValidatorSet, - config: config, - txHandlerFactory: txHandlerFactory, - } -} - -func (ma *inMemoryApp) NextBlock() { - ma.height++ -} - -func (ma *inMemoryApp) Height() int64 { - return ma.height -} - -func (ma *inMemoryApp) RunUpTo(height, index int64) error { - for h := ma.Height(); h <= height; h++ { - block, err := ma.blockstore.GetBlockByHeight(&h) - if err != nil { - return errors.Wrapf(err, "getting block information at height %v", height) - } - for i := 0; i < len(block.Block.Data.Txs); i++ { - if h != height || i != int(index) { - _, _ = ma.ProcessTx(block.Block.Data.Txs[i]) - } else { - return nil - } - } - ma.NextBlock() - } - return errors.Errorf("cannot find transaction at height %d index %d", height, index) -} - -func (ma *inMemoryApp) ProcessTx(txBytes []byte) (txhandler.TxHandlerResult, error) { - return ma.processTx(txBytes, ma.txHandler) -} - -func (ma *inMemoryApp) processTx(txBytes []byte, txHandler txhandler.TxHandler) (txhandler.TxHandlerResult, error) { - splitStoreTx := store.WrapAtomic(ma.store).BeginTx() - defer splitStoreTx.Rollback() - resultBlock, err := ma.blockstore.GetBlockByHeight(&ma.height) - if err != nil { - return txhandler.TxHandlerResult{}, errors.Errorf("retriving block for height %d", ma.height) - } - - storeState := state.NewStoreState2( - context.Background(), - splitStoreTx, - blockHeaderFromHeader(resultBlock.BlockMeta.Header), - ma.getValidatorSet, - ).WithOnChainConfig(ma.config) - return txHandler.ProcessTx(storeState, txBytes, false) -} - -func (ma *inMemoryApp) TraceProcessTx(txBytes []byte, traceCfg eth.TraceConfig) (txhandler.TxHandlerResult, vm.Tracer, error) { - tracer, err := createTracer(traceCfg) - if err != nil { - return txhandler.TxHandlerResult{}, nil, err - } - - // Pass pointer to tracer as we want evm to run with this object, not a copy - traceTxHandle, err := ma.txHandlerFactory.TxHandler(tracer, false) - if err != nil { - return txhandler.TxHandlerResult{}, nil, err - } - res, err := ma.processTx(txBytes, traceTxHandle) - return res, tracer, err -} - -func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { - if traceCfg.Tracer == nil || len(*traceCfg.Tracer) == 0 { - return vm.NewStructLogger(traceCfg.LogConfig), nil - } - tracer, err := tracers.New(*traceCfg.Tracer) - if err != nil { - return nil, err - } - return tracer, nil -} - -func blockHeaderFromHeader(header ttypes.Header) types.BlockHeader { - return types.BlockHeader{ - ChainID: header.ChainID, - Height: header.Height, - Time: int64(header.Time.Unix()), - NumTxs: int32(header.NumTxs), - LastBlockID: types.BlockID{ - Hash: header.LastBlockID.Hash, - }, - ValidatorsHash: header.ValidatorsHash, - AppHash: header.AppHash, - } -} From f7682fe1864adaf2661af93119ce79d078cb2b8f Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 22 Oct 2019 17:44:37 +0100 Subject: [PATCH 21/49] wip --- app.go | 12 +++---- rpc/query_server.go | 6 ++-- store/multi_writer_app_store.go | 57 +-------------------------------- 3 files changed, 10 insertions(+), 65 deletions(-) diff --git a/app.go b/app.go index 490d94cd59..7e322658cf 100644 --- a/app.go +++ b/app.go @@ -611,17 +611,17 @@ func (a *Application) ReadOnlyState() appstate.State { ) } -func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (replay.ReplayApplication, error) { - startVersion := int64(blockNumber) - 1 +func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (replay.ReplayApplication, int64, error) { + startVersion := int64(blockNumber) if startVersion < 0 { - return nil, errors.Errorf("invalid block number %d", blockNumber) + return nil, 0, errors.Errorf("invalid block number %d", blockNumber) } var snapshot store.Snapshot - for ; snapshot != nil || startVersion > 0; snapshot = a.Store.GetSnapshot(startVersion) { + for ; snapshot == nil && startVersion > 0; snapshot = a.Store.GetSnapshot(startVersion) { startVersion-- } if startVersion == 0 { - return nil, errors.Errorf("no saved version for height %d", blockNumber) + return nil, 0, errors.Errorf("no saved version for height %d", blockNumber) } startStore := store.NewSplitStore(snapshot, store.NewMemStore()) @@ -634,5 +634,5 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo a.GetValidatorSet, a.config, a.TxHandlerFactory, - ), nil + ), startVersion, nil } diff --git a/rpc/query_server.go b/rpc/query_server.go index 39ed4c13b0..3578c3bd79 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -63,7 +63,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State - ReplayApplication(uint64, store.BlockStore) (replay.ReplayApplication, error) + ReplayApplication(uint64, store.BlockStore) (replay.ReplayApplication, int64, error) } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -1132,11 +1132,11 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTra return nil, errors.Wrapf(err, "cant parse transaction index %v", receipt.TransactionIndex) } cfg := debug.DecTraceConfig(config) - replayApp, err := s.ReplayApplication(blockNumber, s.BlockStore) + replayApp, startBlockNumber, err := s.ReplayApplication(blockNumber, s.BlockStore) if err != nil { return nil, err } - return debug.TraceTransaction(replayApp, s.BlockStore, 1, int64(blockNumber), int64(txIndex), cfg) + return debug.TraceTransaction(replayApp, s.BlockStore, startBlockNumber, int64(blockNumber), int64(txIndex), cfg) } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index f7740b20ee..c67b1a6daa 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -274,9 +274,8 @@ type multiWriterStoreSnapshot struct { } func newMultiWriterStoreSnapshot(store MultiWriterAppStore, version int64) *multiWriterStoreSnapshot { - evmStore := NewEvmStore(store.evmStore.evmDB, 100) return &multiWriterStoreSnapshot{ - evmDbSnapshot: evmStore, + evmDbSnapshot: NewEvmStore(store.evmStore.evmDB, 100), appStoreSnapshot: store.appStore.GetSnapshot(version), } } @@ -300,7 +299,6 @@ func (s *multiWriterStoreSnapshot) Get(key []byte) []byte { return val } -// Range iterates in-order over the keys in the store prefixed by the given prefix. func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { if len(prefix) == 0 { panic(errors.New("Range over nil prefix not implemented")) @@ -310,57 +308,4 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { return s.evmDbSnapshot.Range(prefix) } return s.appStoreSnapshot.Range(prefix) - /* - // Seems to repeat code in //func (s *EvmStore) Range(prefix []byte) plugin.RangeData ???? - ret := make(plugin.RangeData, 0) - - if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { - it := s.evmDbSnapshot.NewIterator(prefix, prefixRangeEnd(prefix)) - defer it.Close() - - for ; it.Valid(); it.Next() { - key := it.Key() - if util.HasPrefix(key, prefix) { - var err error - key, err = util.UnprefixKey(key, prefix) - if err != nil { - panic(err) - } - - ret = append(ret, &plugin.RangeEntry{ - Key: key, - Value: it.Value(), - }) - } - } - return ret - } - - // Otherwise iterate over the IAVL tree - keys, values, _, err := s.appStoreTree.GetRangeWithProof(prefix, prefixRangeEnd(prefix), 0) - if err != nil { - log.Error("failed to get range", "prefix", string(prefix), "err", err) - return ret - } - - for i, k := range keys { - // Tree range gives all keys that has prefix but it does not check zero byte - // after the prefix. So we have to check zero byte after prefix using util.HasPrefix - if util.HasPrefix(k, prefix) { - k, err = util.UnprefixKey(k, prefix) - if err != nil { - panic(err) - } - } else { // Skip this key as it does not have the prefix - continue - } - - ret = append(ret, &plugin.RangeEntry{ - Key: k, - Value: values[i], - }) - } - - return ret - */ } From d9664f1400025357b4fbac010495182af71e9981 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 23 Oct 2019 11:54:01 +0100 Subject: [PATCH 22/49] merge iAVLVersionReader into iavlStoreSnapshot --- app.go | 10 +++++-- cmd/loom/loom.go | 16 +++++----- replay/replayapp.go | 32 ++++++++++++-------- rpc/debug/trace.go | 4 ++- store/iavlstore.go | 53 +++++++++++++++++++++++---------- store/multi_writer_app_store.go | 17 ++++++++--- store/version_reader.go | 44 --------------------------- store/versioned_cachingstore.go | 8 ++++- 8 files changed, 94 insertions(+), 90 deletions(-) delete mode 100644 store/version_reader.go diff --git a/app.go b/app.go index 7e322658cf..42133aac37 100644 --- a/app.go +++ b/app.go @@ -625,14 +625,18 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo } startStore := store.NewSplitStore(snapshot, store.NewMemStore()) - return replay.NewReplayApplication( + newApp, err := replay.NewReplayApplication( startVersion, blockstore, startStore, - a.TxHandler, a.ReceiptsVersion, a.GetValidatorSet, a.config, a.TxHandlerFactory, - ), startVersion, nil + nil, + ) + if err != nil { + return nil, 0, err + } + return newApp, startVersion, nil } diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index 6592a5c009..d5a43b116c 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -668,15 +668,15 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64) return nil, err } } - - if cfg.CachingStoreConfig.CachingEnabled { - appStore, err = store.NewVersionedCachingStore(appStore, cfg.CachingStoreConfig, appStore.Version()) - if err != nil { - return nil, err + /* + if cfg.CachingStoreConfig.CachingEnabled { + appStore, err = store.NewVersionedCachingStore(appStore, cfg.CachingStoreConfig, appStore.Version()) + if err != nil { + return nil, err + } + logger.Info("VersionedCachingStore enabled") } - logger.Info("VersionedCachingStore enabled") - } - + */ return appStore, nil } diff --git a/replay/replayapp.go b/replay/replayapp.go index 9a59176e54..f3a239cc00 100644 --- a/replay/replayapp.go +++ b/replay/replayapp.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/loomnetwork/go-loom/builtin/types/chainconfig" "github.com/loomnetwork/go-loom/types" + "github.com/pkg/errors" abci "github.com/tendermint/tendermint/abci/types" ttypes "github.com/tendermint/tendermint/types" @@ -16,7 +17,7 @@ import ( type ReplayApplication interface { abci.Application - SetTracer(vm.Tracer) + SetTracer(vm.Tracer) error } var _ abci.Application = (*replayApplication)(nil) @@ -26,38 +27,47 @@ type replayApplication struct { height int64 blockstore store.BlockStore store store.VersionedKVStore - txHandler txhandler.TxHandler receiptsVersion int32 getValidatorSet state.GetValidatorSet config *chainconfig.Config - txHandlerFactory txhandler.TxHandlerFactory tracer vm.Tracer + txHandlerFactory txhandler.TxHandlerFactory + txHandler txhandler.TxHandler } func NewReplayApplication( height int64, blockstore store.BlockStore, store store.VersionedKVStore, - txHandler txhandler.TxHandler, receiptsVersion int32, getValidatorSet state.GetValidatorSet, config *chainconfig.Config, txHandlerFactory txhandler.TxHandlerFactory, -) *replayApplication { - return &replayApplication{ + tracer vm.Tracer, +) (*replayApplication, error) { + app := &replayApplication{ height: height, blockstore: blockstore, store: store, - txHandler: txHandler, receiptsVersion: receiptsVersion, getValidatorSet: getValidatorSet, config: config, txHandlerFactory: txHandlerFactory, } + if err := app.SetTracer(tracer); err != nil { + return nil, err + } + return app, nil } -func (a *replayApplication) SetTracer(tracer vm.Tracer) { +func (a *replayApplication) SetTracer(tracer vm.Tracer) error { + newTxHandle, err := a.txHandlerFactory.TxHandler(tracer, false) + if err != nil { + return errors.Wrap(err, "making transaction handle") + } a.tracer = tracer + a.txHandler = newTxHandle + return nil } func (a *replayApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { @@ -74,12 +84,8 @@ func (a *replayApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { blockHeaderFromHeader(resultBlock.BlockMeta.Header), a.getValidatorSet, ).WithOnChainConfig(a.config) - txHandle, err := a.txHandlerFactory.TxHandler(a.tracer, false) - if err != nil { - return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} - } - r, err := txHandle.ProcessTx(storeState, tx, false) + r, err := a.txHandler.ProcessTx(storeState, tx, false) if err != nil { return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} } diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 929833fcab..5c060e7eb9 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -34,7 +34,9 @@ func TraceTransaction( if err != nil { return nil, err } - app.SetTracer(tracer) + if err := app.SetTracer(tracer); err != nil { + return nil, err + } result := app.DeliverTx(block.Block.Data.Txs[txIndex]) switch tracer := tracer.(type) { diff --git a/store/iavlstore.go b/store/iavlstore.go index e45507d0a8..30b1b52420 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "fmt" "time" @@ -210,20 +211,11 @@ func (s *IAVLStore) Prune() error { } func (s *IAVLStore) GetSnapshot(version int64) Snapshot { - if version == 0 { - // This isn't an actual snapshot obviously, and never will be, but lets pretend... - return &iavlStoreSnapshot{ - IAVLStore: s, - } - } - if s.tree.VersionExists(version) { - reader, err := newIAVLVersionReader(*s, version) - if err != nil { - return nil - } - return reader + snapshot, err := newIavlStoreSnapshot(*s, version) + if err != nil { + panic(err) // todo maybe add error return to Snapshot } - return nil + return snapshot } // NewIAVLStore creates a new IAVLStore. @@ -259,9 +251,38 @@ func NewIAVLStore(db dbm.DB, maxVersions, targetVersion, flushInterval int64) (* } type iavlStoreSnapshot struct { - *IAVLStore + *iavl.ImmutableTree } -func (s *iavlStoreSnapshot) Release() { - // noop +func newIavlStoreSnapshot(store IAVLStore, version int64) (Snapshot, error) { + if version == 0 { + version = store.Version() + } + immutableTree, err := store.tree.GetImmutable(version) + if err != nil { + return nil, err + } + return &iavlStoreSnapshot{ + immutableTree, + }, nil +} + +func (s *iavlStoreSnapshot) Get(key []byte) []byte { + _, value := s.ImmutableTree.Get(key) + return value } + +func (s *iavlStoreSnapshot) Range(prefix []byte) plugin.RangeData { + var data plugin.RangeData + prefix = append(prefix, 0) + s.ImmutableTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { + if 0 != bytes.Compare(prefix, key[:len(prefix)]) { + return true + } + data = append(data, &plugin.RangeEntry{key, value}) + return false + }) + return data +} + +func (s *iavlStoreSnapshot) Release() {} diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index c67b1a6daa..b96863fcd4 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -18,6 +18,10 @@ import ( "github.com/loomnetwork/loomchain/features" ) +const ( + evmCacheRoots = 100 +) + var ( // This is the same prefix as vmPrefix in evm/loomevm.go // We have to do this to avoid cyclic dependency @@ -265,7 +269,8 @@ func (s *MultiWriterAppStore) GetSnapshot(version int64) Snapshot { if version == 0 { version = (*iavl.ImmutableTree)(atomic.LoadPointer(&s.lastSavedTree)).Version() } - return newMultiWriterStoreSnapshot(*s, version) + snapshot, _ := newMultiWriterStoreSnapshot(*s, version) + return snapshot } type multiWriterStoreSnapshot struct { @@ -273,11 +278,15 @@ type multiWriterStoreSnapshot struct { appStoreSnapshot Snapshot } -func newMultiWriterStoreSnapshot(store MultiWriterAppStore, version int64) *multiWriterStoreSnapshot { +func newMultiWriterStoreSnapshot(store MultiWriterAppStore, version int64) (*multiWriterStoreSnapshot, error) { + evmStore := NewEvmStore(store.evmStore.evmDB, evmCacheRoots) + if err := evmStore.LoadVersion(version); err != nil { + return nil, err + } return &multiWriterStoreSnapshot{ - evmDbSnapshot: NewEvmStore(store.evmStore.evmDB, 100), + evmDbSnapshot: evmStore, appStoreSnapshot: store.appStore.GetSnapshot(version), - } + }, nil } func (s *multiWriterStoreSnapshot) Release() { diff --git a/store/version_reader.go b/store/version_reader.go deleted file mode 100644 index bd2d72debc..0000000000 --- a/store/version_reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package store - -import ( - "github.com/loomnetwork/go-loom/plugin" - "github.com/pkg/errors" -) - -type iAVLSnapshot struct { - IAVLStore - version int64 -} - -func newIAVLVersionReader(store IAVLStore, version int64) (Snapshot, error) { - if !store.tree.VersionExists(version) { - return nil, errors.Errorf("version %d missing", version) - } - return &iAVLSnapshot{ - IAVLStore: store, - version: version, - }, nil -} - -func (s *iAVLSnapshot) Get(key []byte) []byte { - value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) - return value -} - -func (s *iAVLSnapshot) Range(prefix []byte) plugin.RangeData { - rangeData := s.IAVLStore.Range(prefix) - var rtv plugin.RangeData - for _, entry := range rangeData { - if s.Has(entry.Key) { - rtv = append(rtv, entry) - } - } - return rtv -} - -func (s *iAVLSnapshot) Has(key []byte) bool { - value, _, _ := s.IAVLStore.tree.GetVersionedWithProof(key, s.version) - return value != nil -} - -func (s *iAVLSnapshot) Release() {} diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 7f19baa266..3c1e267cef 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -374,8 +374,14 @@ func (c *versionedCachingStore) SaveVersion() ([]byte, int64, error) { func (c *versionedCachingStore) GetSnapshot(version int64) Snapshot { return newVersionedCachingStoreSnapshot( c.VersionedKVStore.GetSnapshot(version), - c.cache, c.version-1, c.logger, + c.cache, + version, //c.version-1, + c.logger, ) + //return newVersionedCachingStoreSnapshot( + // c.VersionedKVStore.GetSnapshot(version), + // c.cache, c.version-1, c.logger, + // ) } // CachingStoreSnapshot is a read-only CachingStore with specified version From 0053ff45293b99981812567371518d426b9a591b Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 23 Oct 2019 16:14:50 +0100 Subject: [PATCH 23/49] versionedCachingStore --- store/versioned_cachingstore.go | 35 ++++++++------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 3c1e267cef..9163d5e5a5 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -12,7 +12,6 @@ import ( "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" loom "github.com/loomnetwork/go-loom" - "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -372,16 +371,14 @@ func (c *versionedCachingStore) SaveVersion() ([]byte, int64, error) { } func (c *versionedCachingStore) GetSnapshot(version int64) Snapshot { - return newVersionedCachingStoreSnapshot( - c.VersionedKVStore.GetSnapshot(version), - c.cache, - version, //c.version-1, - c.logger, - ) - //return newVersionedCachingStoreSnapshot( - // c.VersionedKVStore.GetSnapshot(version), - // c.cache, c.version-1, c.logger, - // ) + if version == 0 { + return newVersionedCachingStoreSnapshot( + c.VersionedKVStore.GetSnapshot(version), + c.cache, c.version-1, c.logger, + ) + } else { + return c.VersionedKVStore.GetSnapshot(version) + } } // CachingStoreSnapshot is a read-only CachingStore with specified version @@ -402,14 +399,6 @@ func newVersionedCachingStoreSnapshot(snapshot Snapshot, cache *versionedBigCach } } -func (c *versionedCachingStoreSnapshot) Delete(key []byte) { - panic("[versionedCachingStoreSnapshot] Delete() not implemented") -} - -func (c *versionedCachingStoreSnapshot) Set(key, val []byte) { - panic("[versionedCachingStoreSnapshot] Set() not implemented") -} - func (c *versionedCachingStoreSnapshot) Has(key []byte) bool { var err error @@ -495,14 +484,6 @@ func (c *versionedCachingStoreSnapshot) Get(key []byte) []byte { return data } -func (c *versionedCachingStoreSnapshot) SaveVersion() ([]byte, int64, error) { - return nil, 0, errors.New("[VersionedCachingStoreSnapshot] SaveVersion() not implemented") -} - -func (c *versionedCachingStoreSnapshot) Prune() error { - return errors.New("[VersionedCachingStoreSnapshot] Prune() not implemented") -} - func (c *versionedCachingStoreSnapshot) Release() { c.Snapshot.Release() } From 23746d53f3ab72d6b0a4c07639863b9e1540e4a8 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 23 Oct 2019 16:36:21 +0100 Subject: [PATCH 24/49] PruningIAVLStore --- store/pruning_iavlstore.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 8b4f7b10a7..9ad4ad3bad 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -178,10 +178,7 @@ func (s *PruningIAVLStore) Prune() error { } func (s *PruningIAVLStore) GetSnapshot(version int64) Snapshot { - // This isn't an actual snapshot obviously, and never will be, but lets pretend... - return &pruningIAVLStoreSnapshot{ - PruningIAVLStore: s, - } + return s.store.GetSnapshot(version) } func (s *PruningIAVLStore) prune() error { From c3c0ee4e74589379f3d6407a6d71fa625e5e0136 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 25 Oct 2019 09:10:55 +0100 Subject: [PATCH 25/49] PruningIAVLStore --- cmd/loom/loom.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index d5a43b116c..76462d1954 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -668,15 +668,14 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64) return nil, err } } - /* - if cfg.CachingStoreConfig.CachingEnabled { - appStore, err = store.NewVersionedCachingStore(appStore, cfg.CachingStoreConfig, appStore.Version()) - if err != nil { - return nil, err - } - logger.Info("VersionedCachingStore enabled") + if cfg.CachingStoreConfig.CachingEnabled { + appStore, err = store.NewVersionedCachingStore(appStore, cfg.CachingStoreConfig, appStore.Version()) + if err != nil { + return nil, err } - */ + logger.Info("VersionedCachingStore enabled") + } + return appStore, nil } From e7d01f8456781c89619de52d9b877a110f439161 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 30 Oct 2019 09:13:42 +0000 Subject: [PATCH 26/49] PruningIAVLStore --- e2e/tests/truffle/test/DebugFunctions.js | 76 +++++++++--------------- 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/e2e/tests/truffle/test/DebugFunctions.js b/e2e/tests/truffle/test/DebugFunctions.js index b2b3677d85..a8dae77da3 100644 --- a/e2e/tests/truffle/test/DebugFunctions.js +++ b/e2e/tests/truffle/test/DebugFunctions.js @@ -10,8 +10,6 @@ const { } = require('loom-js'); const TxHashTestContract = artifacts.require('TxHashTestContract'); -const NonceTestContract = artifacts.require('NonceTestContract'); -const SimpleStore = artifacts.require('SimpleStore'); // Requires receipts:v3.3 to be enabled, and receipts:v3.4 not to be, but the new tx hash algo needs // more review & testing before we can release it so skipping this test for now. @@ -44,10 +42,6 @@ contract('TxHashTestContract', async (accounts) => { web3 = new Web3(loomProvider); web3js = new Web3(new Web3.providers.HttpProvider(`http://${nodeAddr}/eth`)); - - //nonceTestContract = await SimpleStore.deployed(); - //nonceContract = new web3.eth.Contract(SimpleStore._json.abi, nonceTestContract.address, {from}); - txHashTestContract = await TxHashTestContract.deployed(); contract = new web3.eth.Contract(TxHashTestContract._json.abi, txHashTestContract.address, {from}); }); @@ -55,51 +49,43 @@ contract('TxHashTestContract', async (accounts) => { it('Test debug_traceTransaction', async () => { console.log("piers contract.methods|", contract.methods); console.log("piers contract.method.set|", contract.methods.set); - - // console.log("piers nonceTestContract.methods|", nonceTestContract.methods); - // console.log("piers nonceTestContract.methods.setValue|", nonceTestContract.methods.setValue); - // console.log("piers nonceTestContract.methods.setValue|", nonceTestContract.methods.setValue(uint256)); try { -/* - const nonceTxResult = await nonceTestContract.methods.set(1111).send(); - console.log("piers nonceTxResult txResult.tx|", nonceTxResult.tx); - console.log("piers nonceTestContract txResult.transactionHash|", nonceTxResult.transactionHash); - await web3js.currentProvider.send({ - method: "debug_traceTransaction", - params: [nonceTxResult.transactionHash], - jsonrpc: "2.0", - id: new Date().getTime() - }, function (error, result) { - console.log("piers!!!!!!!!!nonceTxResult debug_traceTransaction sendResult|", result, "error", error) - console.log("piersn onceTxResult failed|", result.failed); - console.log("piers nonceTxResult structLogs|", result.structLogs); - //assert.equal(true, result === result); - assert.equal(false, result.failed) - }); -*/ + const txResult = await contract.methods.set(1111).send(); - console.log("piers txResult.tx|", txResult.tx); - console.log("piers txResult.transactionHash|", txResult.transactionHash); + //console.log("piers txResult.tx|", txResult.tx); + // console.log("piers txResult.transactionHash|", txResult.transactionHash); + + const receipt = await web3js.eth.getTransactionReceipt(txResult.transactionHash); + console.log("piers receipt|", receipt); + + let blockNumber = await web3js.eth.getBlockNumber(); - await web3js.currentProvider.send({ + console.log("blocknumerb before", blockNumber); + //let bn = blockNumber; + //while ( bn <= blockNumber+3) { + // //await contract.methods.set(1111).send(); + // bn = await web3js.eth.getBlockNumber(); + //console.log("Inloop bn", bn); + //} + await waitForXBlocks(nodeAddr, 5) + blockNumber = await web3js.eth.getBlockNumber(); + console.log("blocknumerb after", blockNumber); + + const waitResult = await web3js.currentProvider.send({ method: "debug_traceTransaction", params: [txResult.transactionHash,{"disableStorage":true,"disableMemory":false,"disableStack":false,"fullStorage":false}], jsonrpc: "2.0", id: new Date().getTime() }, function (error, result) { console.log("piers!!!!!!!!! debug_traceTransaction sendResult|", result, "error", error) - console.log("piers failed|", result.failed); - console.log("piers structLogs|", result.structLogs); + console.log("piers failed|", result.result.failed); + console.log("piers structLogs|", result.result.structLogs); //assert.equal(true, result === result); - assert.equal(undefined, result.failed) + //assert.equal(undefined, result.failed) + assert.equal(true, false); + //let aaa = result.doesnotexist.doesnotexist; }); - //console.log("sendResult3", sendResult3); - - //const doSomething = async () => { - // await sleep2(2000) - // //do stuff - //}; - //console.log("after sleep") + console.log("waitResult", waitResult); const sendResult = await web3js.currentProvider.send({ method: "eth_blockNumber", @@ -111,15 +97,9 @@ contract('TxHashTestContract', async (accounts) => { }); //console.log("sendResult", sendResult); + await waitForXBlocks(nodeAddr, 5) - const receipt = await web3js.eth.getTransactionReceipt(txResult.transactionHash); - //const receipt = await web3js.eth.getTransactionReceipt(txResult.tx); - console.log("piers receipt|", receipt); - - //sleep(1000000) - - - + //sleep1(100000000) } catch(err) { console.log("caught error", err); } From cff6ba6f304f2c82fd3cb784cdcf4a4005726e73 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 30 Oct 2019 12:10:12 +0000 Subject: [PATCH 27/49] reiview issues store changes --- app.go | 6 +-- rpc/debug/trace.go | 1 + store/iavlstore.go | 10 +++-- store/logstore.go | 12 ++++- store/memstore.go | 6 ++- store/multi_writer_app_store.go | 67 ++++++++++++++++++++-------- store/multi_writer_app_store_test.go | 16 +++---- store/pruning_iavlstore.go | 12 ++++- store/splitstore.go | 13 +++++- store/store.go | 3 +- store/store_test.go | 6 +-- store/versioned_cachingstore.go | 16 +++++-- store/versioned_cachingstore_test.go | 18 +++++--- 13 files changed, 133 insertions(+), 53 deletions(-) diff --git a/app.go b/app.go index 42133aac37..a2163ef700 100644 --- a/app.go +++ b/app.go @@ -604,7 +604,7 @@ func (a *Application) ReadOnlyState() appstate.State { // not match the state... need to figure out why this hasn't spectacularly failed already return appstate.NewStoreStateSnapshot( nil, - a.Store.GetSnapshot(0), + a.Store.GetSnapshot(), a.lastBlockHeader, nil, // TODO: last block hash! a.GetValidatorSet, @@ -617,8 +617,8 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo return nil, 0, errors.Errorf("invalid block number %d", blockNumber) } var snapshot store.Snapshot - for ; snapshot == nil && startVersion > 0; snapshot = a.Store.GetSnapshot(startVersion) { - startVersion-- + for err := error(nil); (snapshot == nil || err != nil) && startVersion > 0; startVersion-- { + snapshot, err = a.Store.GetSnapshotAt(startVersion) } if startVersion == 0 { return nil, 0, errors.Errorf("no saved version for height %d", blockNumber) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 5c060e7eb9..0741db7730 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -71,6 +71,7 @@ func runUpTo(app abci.Application, blockstore store.BlockStore, startHeight, hei } _ = app.EndBlock(abci.RequestEndBlock{}) + _ = app.Commit() } return errors.Errorf("cannot find transaction at height %d index %d", height, index) } diff --git a/store/iavlstore.go b/store/iavlstore.go index 30b1b52420..514108b3b4 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -210,14 +210,18 @@ func (s *IAVLStore) Prune() error { return nil } -func (s *IAVLStore) GetSnapshot(version int64) Snapshot { - snapshot, err := newIavlStoreSnapshot(*s, version) +func (s *IAVLStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) if err != nil { - panic(err) // todo maybe add error return to Snapshot + panic(err) } return snapshot } +func (s *IAVLStore) GetSnapshotAt(version int64) (Snapshot, error) { + return newIavlStoreSnapshot(*s, version) +} + // NewIAVLStore creates a new IAVLStore. // maxVersions can be used to specify how many versions should be retained, if set to zero then // old versions will never been deleted. diff --git a/store/logstore.go b/store/logstore.go index 2bbcf72047..9fce729f43 100644 --- a/store/logstore.go +++ b/store/logstore.go @@ -126,6 +126,14 @@ func (s *LogStore) Prune() error { return s.store.Prune() } -func (s *LogStore) GetSnapshot(version int64) Snapshot { - return s.store.GetSnapshot(version) +func (s *LogStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (s *LogStore) GetSnapshotAt(version int64) (Snapshot, error) { + return s.store.GetSnapshotAt(version) } diff --git a/store/memstore.go b/store/memstore.go index 162db180a8..dfe89e143c 100644 --- a/store/memstore.go +++ b/store/memstore.go @@ -76,6 +76,10 @@ func (m *MemStore) Prune() error { return nil } -func (m *MemStore) GetSnapshot(version int64) Snapshot { +func (m *MemStore) GetSnapshot() Snapshot { + panic("not implemented") +} + +func (m *MemStore) GetSnapshotAt(version int64) (Snapshot, error) { panic("not implemented") } diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index b96863fcd4..749744fa23 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -15,6 +15,7 @@ import ( stdprometheus "github.com/prometheus/client_golang/prometheus" "github.com/tendermint/iavl" + "github.com/loomnetwork/loomchain/db" "github.com/loomnetwork/loomchain/features" ) @@ -261,50 +262,62 @@ func (s *MultiWriterAppStore) Prune() error { return s.appStore.Prune() } -func (s *MultiWriterAppStore) GetSnapshot(version int64) Snapshot { +func (s *MultiWriterAppStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (s *MultiWriterAppStore) GetSnapshotAt(version int64) (Snapshot, error) { defer func(begin time.Time) { getSnapshotDuration.Observe(time.Since(begin).Seconds()) }(time.Now()) + var err error + var appStoreTree *iavl.ImmutableTree if version == 0 { - version = (*iavl.ImmutableTree)(atomic.LoadPointer(&s.lastSavedTree)).Version() + appStoreTree = (*iavl.ImmutableTree)(atomic.LoadPointer(&s.lastSavedTree)) + } else { + appStoreTree, err = s.appStore.tree.GetImmutable(version) + if err != nil { + return nil, errors.Wrapf(err, "failed to load immutable tree for version %v", version) + } } - snapshot, _ := newMultiWriterStoreSnapshot(*s, version) - return snapshot + evmDbSnapshot := s.evmStore.GetSnapshot(appStoreTree.Version()) + return newMultiWriterStoreSnapshot(evmDbSnapshot, appStoreTree), nil } type multiWriterStoreSnapshot struct { - evmDbSnapshot KVReader - appStoreSnapshot Snapshot + evmDbSnapshot db.Snapshot + appStoreTree *iavl.ImmutableTree } -func newMultiWriterStoreSnapshot(store MultiWriterAppStore, version int64) (*multiWriterStoreSnapshot, error) { - evmStore := NewEvmStore(store.evmStore.evmDB, evmCacheRoots) - if err := evmStore.LoadVersion(version); err != nil { - return nil, err - } +func newMultiWriterStoreSnapshot(evmDbSnapshot db.Snapshot, appStoreTree *iavl.ImmutableTree) *multiWriterStoreSnapshot { return &multiWriterStoreSnapshot{ - evmDbSnapshot: evmStore, - appStoreSnapshot: store.appStore.GetSnapshot(version), - }, nil + evmDbSnapshot: evmDbSnapshot, + appStoreTree: appStoreTree, + } } func (s *multiWriterStoreSnapshot) Release() { - s.appStoreSnapshot.Release() + s.evmDbSnapshot.Release() + s.appStoreTree = nil } func (s *multiWriterStoreSnapshot) Has(key []byte) bool { if util.HasPrefix(key, vmPrefix) { return s.evmDbSnapshot.Has(key) } - return s.appStoreSnapshot.Has(key) + return s.appStoreTree.Has(key) } func (s *multiWriterStoreSnapshot) Get(key []byte) []byte { if util.HasPrefix(key, vmPrefix) { return s.evmDbSnapshot.Get(key) } - val := s.appStoreSnapshot.Get(key) + _, val := s.appStoreTree.Get(key) return val } @@ -313,8 +326,24 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { panic(errors.New("Range over nil prefix not implemented")) } + var data plugin.RangeData if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { - return s.evmDbSnapshot.Range(prefix) + for iter := s.evmDbSnapshot.NewIterator(prefix, nil); iter.Valid(); iter.Next() { + if 0 != bytes.Compare(prefix, iter.Key()[:len(prefix)]) { + break + } + data = append(data, &plugin.RangeEntry{iter.Key(), iter.Value()}) + } + } else { + prefix = append(prefix, 0) + s.appStoreTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { + if 0 != bytes.Compare(prefix, key[:len(prefix)]) { + return true + } + data = append(data, &plugin.RangeEntry{key, value}) + return false + }) } - return s.appStoreSnapshot.Range(prefix) + + return data } diff --git a/store/multi_writer_app_store_test.go b/store/multi_writer_app_store_test.go index ce311ca558..f08d244716 100644 --- a/store/multi_writer_app_store_test.go +++ b/store/multi_writer_app_store_test.go @@ -112,7 +112,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShot() { store.Set(vmPrefixKey("abcde"), []byte("vvvvvvvvv")) store.Set([]byte("abcd"), []byte("asdfasdf")) - snapshot := store.GetSnapshot(0) + snapshot := store.GetSnapshot() require.Equal([]byte("hello"), snapshot.Get(vmPrefixKey("abcd"))) require.Equal([]byte("NewData"), snapshot.Get([]byte("abcd"))) require.Equal([]byte("world"), snapshot.Get(vmPrefixKey("abcde"))) @@ -120,7 +120,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShot() { _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot(0) + snapshot = store.GetSnapshot() require.Equal([]byte("asdfasdf"), snapshot.Get([]byte("abcd"))) require.Equal([]byte("hellooooooo"), snapshot.Get(vmPrefixKey("abcd"))) require.Equal([]byte("vvvvvvvvv"), snapshot.Get(vmPrefixKey("abcde"))) @@ -143,7 +143,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotFlushInter store.Set([]byte("test2"), []byte("test2v2")) // this snapshot is from memory - snapshotv1 := store.GetSnapshot(0) + snapshotv1 := store.GetSnapshot() require.Equal([]byte("test1"), snapshotv1.Get([]byte("test1"))) require.Equal([]byte("test2"), snapshotv1.Get([]byte("test2"))) @@ -152,7 +152,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotFlushInter require.NoError(err) // get snapshotv2 - snapshotv2 := store.GetSnapshot(0) + snapshotv2 := store.GetSnapshot() require.Equal([]byte("test1v2"), snapshotv2.Get([]byte("test1"))) require.Equal([]byte("test2v2"), snapshotv2.Get([]byte("test2"))) @@ -176,13 +176,13 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { store.Set([]byte("uuuu"), []byte("SSSSSSSSSSSSS")) store.Set([]byte("sssss"), []byte("NewData")) - snapshot := store.GetSnapshot(0) + snapshot := store.GetSnapshot() rangeData := snapshot.Range(vmPrefix) require.Equal(0, len(rangeData)) _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot(0) + snapshot = store.GetSnapshot() rangeData = snapshot.Range(vmPrefix) require.Equal(4+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, bytes.Compare(snapshot.Get(vmPrefixKey("abcd")), []byte("hello"))) @@ -194,7 +194,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { store.Delete(vmPrefixKey("abcd")) store.Delete([]byte("ssssvvv")) - snapshot = store.GetSnapshot(0) + snapshot = store.GetSnapshot() rangeData = snapshot.Range(vmPrefix) require.Equal(4+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, bytes.Compare(snapshot.Get(vmPrefixKey("abcd")), []byte("hello"))) @@ -205,7 +205,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSnapShotRange() { _, _, err = store.SaveVersion() require.NoError(err) - snapshot = store.GetSnapshot(0) + snapshot = store.GetSnapshot() rangeData = snapshot.Range(vmPrefix) require.Equal(3+1, len(rangeData)) // +1 for evm root stored by EVM store require.Equal(0, len(snapshot.Get(vmPrefixKey("abcd")))) // has been deleted diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 9ad4ad3bad..7f452f445d 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -177,8 +177,16 @@ func (s *PruningIAVLStore) Prune() error { return nil } -func (s *PruningIAVLStore) GetSnapshot(version int64) Snapshot { - return s.store.GetSnapshot(version) +func (s *PruningIAVLStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (s *PruningIAVLStore) GetSnapshotAt(version int64) (Snapshot, error) { + return s.store.GetSnapshotAt(version) } func (s *PruningIAVLStore) prune() error { diff --git a/store/splitstore.go b/store/splitstore.go index aec422be30..0096e7ea9d 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -73,6 +73,15 @@ func (ss splitStore) SaveVersion() ([]byte, int64, error) { func (ss splitStore) Prune() error { return errors.New("not implemented") } -func (ss splitStore) GetSnapshot(version int64) Snapshot { - return nil + +func (s *splitStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (ss splitStore) GetSnapshotAt(version int64) (Snapshot, error) { + return nil, nil } diff --git a/store/store.go b/store/store.go index 167b19254b..85e0c7fe72 100644 --- a/store/store.go +++ b/store/store.go @@ -53,7 +53,8 @@ type VersionedKVStore interface { SaveVersion() ([]byte, int64, error) // Delete old version of the store Prune() error - GetSnapshot(version int64) Snapshot + GetSnapshot() Snapshot + GetSnapshotAt(version int64) (Snapshot, error) } type cacheItem struct { diff --git a/store/store_test.go b/store/store_test.go index 941c0b5b0f..40065bff2e 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -348,7 +348,7 @@ func (ts *StoreTestSuite) VerifyConcurrentSnapshots() { if snap != nil { snap.Release() } - snap = ts.store.GetSnapshot(0) + snap = ts.store.GetSnapshot() } snap.Get([]byte(fmt.Sprintf("key/%d", i))) } @@ -395,7 +395,7 @@ func (ts *IAVLStoreTestSuite) TestSnapshotRange() { // but this store doesn't have real snapshots so the snapshot is expected to contain the same // unsaved state as the store itself... func() { - snap := ts.store.GetSnapshot(0) + snap := ts.store.GetSnapshot() defer snap.Release() ts.VerifyRange(snap, prefixes, entries) @@ -405,7 +405,7 @@ func (ts *IAVLStoreTestSuite) TestSnapshotRange() { // snapshot should see all the data that was saved to disk func() { - snap := ts.store.GetSnapshot(0) + snap := ts.store.GetSnapshot() defer snap.Release() ts.VerifyRange(snap, prefixes, entries) diff --git a/store/versioned_cachingstore.go b/store/versioned_cachingstore.go index 9163d5e5a5..e7ad896228 100644 --- a/store/versioned_cachingstore.go +++ b/store/versioned_cachingstore.go @@ -370,14 +370,22 @@ func (c *versionedCachingStore) SaveVersion() ([]byte, int64, error) { return hash, version, err } -func (c *versionedCachingStore) GetSnapshot(version int64) Snapshot { +func (s *versionedCachingStore) GetSnapshot() Snapshot { + snapshot, err := s.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (c *versionedCachingStore) GetSnapshotAt(version int64) (Snapshot, error) { if version == 0 { return newVersionedCachingStoreSnapshot( - c.VersionedKVStore.GetSnapshot(version), + c.VersionedKVStore.GetSnapshot(), c.cache, c.version-1, c.logger, - ) + ), nil } else { - return c.VersionedKVStore.GetSnapshot(version) + return c.VersionedKVStore.GetSnapshotAt(version) } } diff --git a/store/versioned_cachingstore_test.go b/store/versioned_cachingstore_test.go index a332d5bab0..e00095e1a5 100644 --- a/store/versioned_cachingstore_test.go +++ b/store/versioned_cachingstore_test.go @@ -58,7 +58,15 @@ func (m *MockStore) Prune() error { return nil } -func (m *MockStore) GetSnapshot(version int64) Snapshot { +func (m *MockStore) GetSnapshot() Snapshot { + snapshot, err := m.GetSnapshotAt(0) + if err != nil { + panic(err) + } + return snapshot +} + +func (m *MockStore) GetSnapshotAt(version int64) (Snapshot, error) { snapshotStore := make(map[string][]byte) for k, v := range m.storage { snapshotStore[k] = v @@ -68,7 +76,7 @@ func (m *MockStore) GetSnapshot(version int64) Snapshot { } return &mockStoreSnapshot{ MockStore: mstore, - } + }, nil } type mockStoreSnapshot struct { @@ -98,7 +106,7 @@ func TestCachingStoreVersion(t *testing.T) { mockStore.Set(key2, []byte("value2")) mockStore.Set(key3, []byte("value3")) - snapshotv0 := cachingStore.GetSnapshot(0) + snapshotv0 := cachingStore.GetSnapshot() // cachingStoreSnapshot will cache key1 in memory as version 0 cachedValue := snapshotv0.Get(key1) @@ -114,7 +122,7 @@ func TestCachingStoreVersion(t *testing.T) { // save data into version 1 cachingStore.Set(key2, []byte("newvalue2")) cachingStore.Set(key3, []byte("newvalue3")) - snapshotv1 := cachingStore.GetSnapshot(0) + snapshotv1 := cachingStore.GetSnapshot() cachedValue = snapshotv1.Get(key2) assert.Equal(t, "newvalue2", string(cachedValue), "snapshotv1 should get correct value") cachedValue = snapshotv1.Get(key1) @@ -136,7 +144,7 @@ func TestCachingStoreVersion(t *testing.T) { // save to bump up version _, version, _ = cachingStore.SaveVersion() assert.Equal(t, int64(2), version, "version must be updated to 2") - snapshotv2 := cachingStore.GetSnapshot(0) + snapshotv2 := cachingStore.GetSnapshot() cachedValue = snapshotv2.Get(key1) assert.Equal(t, "value1", string(cachedValue), "snapshotv2 should get the value from cache") cachedValue = snapshotv2.Get(key2) From f2323ae81b27012d7ee99ca5fbc1fc5b1f1fa6ba Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 30 Oct 2019 15:20:39 +0000 Subject: [PATCH 28/49] use loomchain.Application --- app.go | 56 ++++++++++++------ replay/replayapp.go | 140 -------------------------------------------- rpc/debug/trace.go | 35 ++++++++--- rpc/query_server.go | 5 +- 4 files changed, 68 insertions(+), 168 deletions(-) delete mode 100644 replay/replayapp.go diff --git a/app.go b/app.go index a2163ef700..ee8906cdff 100644 --- a/app.go +++ b/app.go @@ -7,12 +7,14 @@ import ( "fmt" "time" + "github.com/ethereum/go-ethereum/core/vm" "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" + "github.com/loomnetwork/go-loom" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" "github.com/pkg/errors" - //"github.com/pkg/errors" + // "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" abci "github.com/tendermint/tendermint/abci/types" ttypes "github.com/tendermint/tendermint/types" @@ -20,7 +22,6 @@ import ( "github.com/loomnetwork/loomchain/eth/utils" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/registry" - "github.com/loomnetwork/loomchain/replay" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/log" @@ -550,9 +551,11 @@ func (a *Application) Commit() abci.ResponseCommit { height := a.curBlockHeader.GetHeight() - if err := a.EvmAuxStore.SaveChildTxRefs(a.childTxRefs); err != nil { - // TODO: consider panic instead - log.Error("Failed to save Tendermint -> EVM tx hash refs", "height", height, "err", err) + if a.EvmAuxStore != nil { + if err := a.EvmAuxStore.SaveChildTxRefs(a.childTxRefs); err != nil { + // TODO: consider panic instead + log.Error("Failed to save Tendermint -> EVM tx hash refs", "height", height, "err", err) + } } a.childTxRefs = nil @@ -611,7 +614,7 @@ func (a *Application) ReadOnlyState() appstate.State { ) } -func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (replay.ReplayApplication, int64, error) { +func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (*Application, int64, error) { startVersion := int64(blockNumber) if startVersion < 0 { return nil, 0, errors.Errorf("invalid block number %d", blockNumber) @@ -624,19 +627,38 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo return nil, 0, errors.Errorf("no saved version for height %d", blockNumber) } - startStore := store.NewSplitStore(snapshot, store.NewMemStore()) - newApp, err := replay.NewReplayApplication( - startVersion, - blockstore, - startStore, - a.ReceiptsVersion, - a.GetValidatorSet, - a.config, - a.TxHandlerFactory, - nil, - ) + txHandle, err := a.TxHandlerFactory.TxHandler(nil, false) if err != nil { return nil, 0, err } + newApp := &Application{ + Store: store.NewSplitStore(snapshot, store.NewMemStore()), + Init: func(state appstate.State) error { + panic("init should not be called") + }, + TxHandler: txHandle, + TxHandlerFactory: a.TxHandlerFactory, + BlockIndexStore: nil, + EventHandler: nil, + ReceiptHandlerProvider: nil, + CreateValidatorManager: nil, + CreateChainConfigManager: nil, + CreateContractUpkeepHandler: nil, + EventStore: nil, + GetValidatorSet: func(state appstate.State) (loom.ValidatorSet, error) { + return loom.NewValidatorSet(), nil + }, + EvmAuxStore: nil, + ReceiptsVersion: a.ReceiptsVersion, + } return newApp, startVersion, nil } + +func (a *Application) SetTracer(tracer vm.Tracer, metrics bool) error { + newTxHandle, err := a.TxHandlerFactory.TxHandler(tracer, metrics) + if err != nil { + return errors.Wrap(err, "making transaction handle") + } + a.TxHandler = newTxHandle + return nil +} diff --git a/replay/replayapp.go b/replay/replayapp.go deleted file mode 100644 index f3a239cc00..0000000000 --- a/replay/replayapp.go +++ /dev/null @@ -1,140 +0,0 @@ -package replay - -import ( - "context" - - "github.com/ethereum/go-ethereum/core/vm" - "github.com/loomnetwork/go-loom/builtin/types/chainconfig" - "github.com/loomnetwork/go-loom/types" - "github.com/pkg/errors" - abci "github.com/tendermint/tendermint/abci/types" - ttypes "github.com/tendermint/tendermint/types" - - "github.com/loomnetwork/loomchain/state" - "github.com/loomnetwork/loomchain/store" - "github.com/loomnetwork/loomchain/txhandler" -) - -type ReplayApplication interface { - abci.Application - SetTracer(vm.Tracer) error -} - -var _ abci.Application = (*replayApplication)(nil) -var _ abci.Application = ReplayApplication(nil) - -type replayApplication struct { - height int64 - blockstore store.BlockStore - store store.VersionedKVStore - receiptsVersion int32 - getValidatorSet state.GetValidatorSet - config *chainconfig.Config - tracer vm.Tracer - txHandlerFactory txhandler.TxHandlerFactory - txHandler txhandler.TxHandler -} - -func NewReplayApplication( - height int64, - blockstore store.BlockStore, - store store.VersionedKVStore, - receiptsVersion int32, - getValidatorSet state.GetValidatorSet, - config *chainconfig.Config, - txHandlerFactory txhandler.TxHandlerFactory, - tracer vm.Tracer, -) (*replayApplication, error) { - app := &replayApplication{ - height: height, - blockstore: blockstore, - store: store, - receiptsVersion: receiptsVersion, - getValidatorSet: getValidatorSet, - config: config, - txHandlerFactory: txHandlerFactory, - } - if err := app.SetTracer(tracer); err != nil { - return nil, err - } - return app, nil -} - -func (a *replayApplication) SetTracer(tracer vm.Tracer) error { - newTxHandle, err := a.txHandlerFactory.TxHandler(tracer, false) - if err != nil { - return errors.Wrap(err, "making transaction handle") - } - a.tracer = tracer - a.txHandler = newTxHandle - return nil -} - -func (a *replayApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { - splitStoreTx := store.WrapAtomic(a.store).BeginTx() - defer splitStoreTx.Rollback() - resultBlock, err := a.blockstore.GetBlockByHeight(&a.height) - if err != nil { - return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} - } - - storeState := state.NewStoreState2( - context.Background(), - splitStoreTx, - blockHeaderFromHeader(resultBlock.BlockMeta.Header), - a.getValidatorSet, - ).WithOnChainConfig(a.config) - - r, err := a.txHandler.ProcessTx(storeState, tx, false) - if err != nil { - return abci.ResponseDeliverTx{Code: 1, Log: err.Error()} - } - return abci.ResponseDeliverTx{Code: abci.CodeTypeOK, Data: r.Data, Tags: r.Tags, Info: r.Info} -} - -func (_ *replayApplication) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginBlock { - return abci.ResponseBeginBlock{} -} - -func (a *replayApplication) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock { - a.height++ - return abci.ResponseEndBlock{} -} - -func (_ *replayApplication) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { - return abci.ResponseInitChain{} -} - -func (_ *replayApplication) Info(req abci.RequestInfo) abci.ResponseInfo { - return abci.ResponseInfo{} -} - -func (_ *replayApplication) SetOption(req abci.RequestSetOption) abci.ResponseSetOption { - return abci.ResponseSetOption{} -} - -func (_ *replayApplication) CheckTx(tx []byte) abci.ResponseCheckTx { - return abci.ResponseCheckTx{Code: 0} -} - -func (_ *replayApplication) Commit() abci.ResponseCommit { - return abci.ResponseCommit{} -} - -func (_ *replayApplication) Query(req abci.RequestQuery) abci.ResponseQuery { - return abci.ResponseQuery{Code: 0} -} - -func blockHeaderFromHeader(header ttypes.Header) types.BlockHeader { - return types.BlockHeader{ - ChainID: header.ChainID, - Height: header.Height, - Time: int64(header.Time.Unix()), - NumTxs: int32(header.NumTxs), - LastBlockID: types.BlockID{ - Hash: header.LastBlockID.Hash, - }, - ValidatorsHash: header.ValidatorsHash, - AppHash: header.AppHash, - } -} diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 0741db7730..a43d90a16d 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -7,17 +7,19 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethapi" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/pkg/errors" abci "github.com/tendermint/tendermint/abci/types" + ctypes "github.com/tendermint/tendermint/rpc/core/types" + ttypes "github.com/tendermint/tendermint/types" - "github.com/ethereum/go-ethereum/eth/ethapi" - "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/loomnetwork/loomchain/replay" + "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/store" ) func TraceTransaction( - app replay.ReplayApplication, + app loomchain.Application, blockstore store.BlockStore, startBlockNumber, targetBlockNumber, txIndex int64, config eth.TraceConfig, @@ -34,7 +36,7 @@ func TraceTransaction( if err != nil { return nil, err } - if err := app.SetTracer(tracer); err != nil { + if err := app.SetTracer(tracer, false); err != nil { return nil, err } result := app.DeliverTx(block.Block.Data.Txs[txIndex]) @@ -54,9 +56,13 @@ func TraceTransaction( } } -func runUpTo(app abci.Application, blockstore store.BlockStore, startHeight, height, index int64) error { +func runUpTo(app loomchain.Application, blockstore store.BlockStore, startHeight, height, index int64) error { for h := startHeight; h <= height; h++ { - _ = app.BeginBlock(abci.RequestBeginBlock{}) + resultBlock, err := blockstore.GetBlockByHeight(&h) + if err != nil { + return err + } + _ = app.BeginBlock(requestBeginBlock(*resultBlock)) block, err := blockstore.GetBlockByHeight(&h) if err != nil { @@ -70,12 +76,25 @@ func runUpTo(app abci.Application, blockstore store.BlockStore, startHeight, hei } } - _ = app.EndBlock(abci.RequestEndBlock{}) + _ = app.EndBlock(requestEndBlock(h)) _ = app.Commit() } return errors.Errorf("cannot find transaction at height %d index %d", height, index) } +func requestBeginBlock(resultBlock ctypes.ResultBlock) abci.RequestBeginBlock { + return abci.RequestBeginBlock{ + Header: ttypes.TM2PB.Header(&resultBlock.BlockMeta.Header), + Hash: resultBlock.BlockMeta.BlockID.Hash, + } +} + +func requestEndBlock(height int64) abci.RequestEndBlock { + return abci.RequestEndBlock{ + Height: height, + } +} + func createTracer(traceCfg eth.TraceConfig) (vm.Tracer, error) { if traceCfg.Tracer == nil || len(*traceCfg.Tracer) == 0 { return vm.NewStructLogger(traceCfg.LogConfig), nil diff --git a/rpc/query_server.go b/rpc/query_server.go index 3578c3bd79..1fc40179c4 100755 --- a/rpc/query_server.go +++ b/rpc/query_server.go @@ -39,7 +39,6 @@ import ( "github.com/loomnetwork/loomchain/receipts/common" "github.com/loomnetwork/loomchain/registry" registryFac "github.com/loomnetwork/loomchain/registry/factory" - "github.com/loomnetwork/loomchain/replay" "github.com/loomnetwork/loomchain/rpc/debug" "github.com/loomnetwork/loomchain/rpc/eth" appstate "github.com/loomnetwork/loomchain/state" @@ -63,7 +62,7 @@ const ( // StateProvider interface is used by QueryServer to access the read-only application state type StateProvider interface { ReadOnlyState() appstate.State - ReplayApplication(uint64, store.BlockStore) (replay.ReplayApplication, int64, error) + ReplayApplication(uint64, store.BlockStore) (*loomchain.Application, int64, error) } // QueryServer provides the ability to query the current state of the DAppChain via RPC. @@ -1136,7 +1135,7 @@ func (s *QueryServer) DebugTraceTransaction(hash eth.Data, config *debug.JsonTra if err != nil { return nil, err } - return debug.TraceTransaction(replayApp, s.BlockStore, startBlockNumber, int64(blockNumber), int64(txIndex), cfg) + return debug.TraceTransaction(*replayApp, s.BlockStore, startBlockNumber, int64(blockNumber), int64(txIndex), cfg) } func (s *QueryServer) getBlockHeightFromHash(hash []byte) (uint64, error) { From 3f0d0069448cabf01fa9b03aa9dd3517b9078a50 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 30 Oct 2019 18:23:41 +0000 Subject: [PATCH 29/49] loomchain.Application --- app.go | 22 ++++++++++++---------- store/splitstore.go | 28 ++++++++++++++++------------ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/app.go b/app.go index ee8906cdff..a0f3943136 100644 --- a/app.go +++ b/app.go @@ -211,7 +211,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB a.curBlockHeader = block a.curBlockHash = req.Hash - + log.Info("piers bock hash and height", "height", block.Height, "hash", string(req.Hash)) if a.CreateContractUpkeepHandler != nil { upkeepStoreTx := store.WrapAtomic(a.Store).BeginTx() upkeepState := appstate.NewStoreState( @@ -632,18 +632,20 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo return nil, 0, err } newApp := &Application{ - Store: store.NewSplitStore(snapshot, store.NewMemStore()), + Store: store.NewSplitStore(snapshot, store.NewMemStore(), startVersion-1), Init: func(state appstate.State) error { panic("init should not be called") }, - TxHandler: txHandle, - TxHandlerFactory: a.TxHandlerFactory, - BlockIndexStore: nil, - EventHandler: nil, - ReceiptHandlerProvider: nil, - CreateValidatorManager: nil, - CreateChainConfigManager: nil, - CreateContractUpkeepHandler: nil, + TxHandler: txHandle, + TxHandlerFactory: a.TxHandlerFactory, + BlockIndexStore: nil, + EventHandler: nil, + ReceiptHandlerProvider: nil, + CreateValidatorManager: func(state appstate.State) (ValidatorsManager, error) { + return nil, registry.ErrNotFound + }, + CreateChainConfigManager: a.CreateChainConfigManager, + CreateContractUpkeepHandler: a.CreateContractUpkeepHandler, EventStore: nil, GetValidatorSet: func(state appstate.State) (loom.ValidatorSet, error) { return loom.NewValidatorSet(), nil diff --git a/store/splitstore.go b/store/splitstore.go index 0096e7ea9d..5540a178f4 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -2,20 +2,21 @@ package store import ( "github.com/loomnetwork/go-loom/plugin" - "github.com/pkg/errors" ) type splitStore struct { KVReader VersionedKVStore deleted map[string]bool + version int64 } -func NewSplitStore(full KVReader, empty VersionedKVStore) VersionedKVStore { +func NewSplitStore(full KVReader, empty VersionedKVStore, version int64) VersionedKVStore { return &splitStore{ KVReader: full, VersionedKVStore: empty, deleted: make(map[string]bool), + version: version, } } @@ -64,24 +65,27 @@ func (ss splitStore) Hash() []byte { return nil } func (ss splitStore) Version() int64 { - return 0 + return ss.version } func (ss splitStore) SaveVersion() ([]byte, int64, error) { - return nil, 0, nil + ss.version++ + return nil, ss.version, nil } func (ss splitStore) Prune() error { - return errors.New("not implemented") + panic("should not be called") } -func (s *splitStore) GetSnapshot() Snapshot { - snapshot, err := s.GetSnapshotAt(0) - if err != nil { - panic(err) - } - return snapshot +func (ss *splitStore) GetSnapshot() Snapshot { + return splitStoreSnapShot{*ss} } func (ss splitStore) GetSnapshotAt(version int64) (Snapshot, error) { - return nil, nil + panic("should not be called") +} + +type splitStoreSnapShot struct { + splitStore } + +func (ss splitStoreSnapShot) Release() {} From 39d948d4188b4fbed8f7d616757b035bb9bcdcb9 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 10:04:56 +0000 Subject: [PATCH 30/49] Add valiadtor functions, fix range --- app.go | 25 ++++++++++--------------- store/iavlstore.go | 2 +- store/multi_writer_app_store.go | 2 +- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/app.go b/app.go index a0f3943136..b6095c1bb0 100644 --- a/app.go +++ b/app.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/go-kit/kit/metrics" kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/loomnetwork/go-loom" cctypes "github.com/loomnetwork/go-loom/builtin/types/chainconfig" "github.com/pkg/errors" @@ -211,7 +210,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB a.curBlockHeader = block a.curBlockHash = req.Hash - log.Info("piers bock hash and height", "height", block.Height, "hash", string(req.Hash)) + if a.CreateContractUpkeepHandler != nil { upkeepStoreTx := store.WrapAtomic(a.Store).BeginTx() upkeepState := appstate.NewStoreState( @@ -636,22 +635,18 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo Init: func(state appstate.State) error { panic("init should not be called") }, - TxHandler: txHandle, - TxHandlerFactory: a.TxHandlerFactory, - BlockIndexStore: nil, - EventHandler: nil, - ReceiptHandlerProvider: nil, - CreateValidatorManager: func(state appstate.State) (ValidatorsManager, error) { - return nil, registry.ErrNotFound - }, + TxHandler: txHandle, + TxHandlerFactory: a.TxHandlerFactory, + BlockIndexStore: nil, + EventHandler: nil, + ReceiptHandlerProvider: nil, + CreateValidatorManager: a.CreateValidatorManager, CreateChainConfigManager: a.CreateChainConfigManager, CreateContractUpkeepHandler: a.CreateContractUpkeepHandler, EventStore: nil, - GetValidatorSet: func(state appstate.State) (loom.ValidatorSet, error) { - return loom.NewValidatorSet(), nil - }, - EvmAuxStore: nil, - ReceiptsVersion: a.ReceiptsVersion, + GetValidatorSet: a.GetValidatorSet, + EvmAuxStore: nil, + ReceiptsVersion: a.ReceiptsVersion, } return newApp, startVersion, nil } diff --git a/store/iavlstore.go b/store/iavlstore.go index 514108b3b4..7ea1ecb5a2 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -280,7 +280,7 @@ func (s *iavlStoreSnapshot) Range(prefix []byte) plugin.RangeData { var data plugin.RangeData prefix = append(prefix, 0) s.ImmutableTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { - if 0 != bytes.Compare(prefix, key[:len(prefix)]) { + if len(key) < len(prefix) || 0 != bytes.Compare(prefix, key[:len(prefix)]) { return true } data = append(data, &plugin.RangeEntry{key, value}) diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index 749744fa23..a2fd6475ce 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -337,7 +337,7 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { } else { prefix = append(prefix, 0) s.appStoreTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { - if 0 != bytes.Compare(prefix, key[:len(prefix)]) { + if len(key) < len(prefix) || 0 != bytes.Compare(prefix, key[:len(prefix)]) { return true } data = append(data, &plugin.RangeEntry{key, value}) From 5de9514f938b4685eef932b2bb9d93710be8271e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 13:24:53 +0000 Subject: [PATCH 31/49] fx deploy fixes --- app.go | 67 +++++++++++++++++++++++++-------------------- rpc/debug/trace.go | 12 ++++++-- store/splitstore.go | 26 +++++++++--------- 3 files changed, 60 insertions(+), 45 deletions(-) diff --git a/app.go b/app.go index b6095c1bb0..5ede30ae00 100644 --- a/app.go +++ b/app.go @@ -291,13 +291,16 @@ func (a *Application) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock { // TODO: receiptHandler.CommitBlock() should be moved to Application.Commit() storeTx := store.WrapAtomic(a.Store).BeginTx() - receiptHandler := a.ReceiptHandlerProvider.Store() - if err := receiptHandler.CommitBlock(a.height()); err != nil { - storeTx.Rollback() - // TODO: maybe panic instead? - log.Error(fmt.Sprintf("aborted committing block receipts, %v", err.Error())) - } else { - storeTx.Commit() + + if a.ReceiptHandlerProvider != nil { + receiptHandler := a.ReceiptHandlerProvider.Store() + if err := receiptHandler.CommitBlock(a.height()); err != nil { + storeTx.Rollback() + // TODO: maybe panic instead? + log.Error(fmt.Sprintf("aborted committing block receipts, %v", err.Error())) + } else { + storeTx.Commit() + } } storeTx = store.WrapAtomic(a.Store).BeginTx() @@ -486,16 +489,17 @@ func (a *Application) deliverTx2(storeTx store.KVStoreTx, txBytes []byte) abci.R a.curBlockHash, a.GetValidatorSet, ).WithOnChainConfig(a.config) - - receiptHandler := a.ReceiptHandlerProvider.Store() - defer receiptHandler.DiscardCurrentReceipt() - defer a.EventHandler.Rollback() + if a.ReceiptHandlerProvider != nil { + //receiptHandler := a.ReceiptHandlerProvider.Store() + defer a.ReceiptHandlerProvider.Store().DiscardCurrentReceipt() + defer a.EventHandler.Rollback() + } r, txErr := a.TxHandler.ProcessTx(state, txBytes, false) // Store the receipt even if the tx itself failed var receiptTxHash []byte - if a.ReceiptHandlerProvider.Reader().GetCurrentReceipt() != nil { + if a.ReceiptHandlerProvider != nil && a.ReceiptHandlerProvider.Reader().GetCurrentReceipt() != nil { receiptTxHash = a.ReceiptHandlerProvider.Reader().GetCurrentReceipt().TxHash txHash := ttypes.Tx(txBytes).Hash() // If a receipt was generated for an EVM tx add a link between the TM tx hash and the EVM tx hash @@ -506,7 +510,7 @@ func (a *Application) deliverTx2(storeTx store.KVStoreTx, txBytes []byte) abci.R ChildTxHash: receiptTxHash, }) } - receiptHandler.CommitCurrentReceipt() + a.ReceiptHandlerProvider.Store().CommitCurrentReceipt() } if txErr != nil { @@ -517,13 +521,15 @@ func (a *Application) deliverTx2(storeTx store.KVStoreTx, txBytes []byte) abci.R return abci.ResponseDeliverTx{Code: 1, Data: r.Data, Log: txErr.Error()} } - a.EventHandler.Commit(uint64(a.curBlockHeader.GetHeight())) storeTx.Commit() - // FIXME: Really shouldn't be sending out events until the whole block is committed because - // the state changes from the tx won't be visible to queries until after Application.Commit() - if err := a.EventHandler.LegacyEthSubscriptionSet().EmitTxEvent(r.Data, r.Info); err != nil { - log.Error("Emit Tx Event error", "err", err) + if a.EventHandler != nil { + a.EventHandler.Commit(uint64(a.curBlockHeader.GetHeight())) + // FIXME: Really shouldn't be sending out events until the whole block is committed because + // the state changes from the tx won't be visible to queries until after Application.Commit() + if err := a.EventHandler.LegacyEthSubscriptionSet().EmitTxEvent(r.Data, r.Info); err != nil { + log.Error("Emit Tx Event error", "err", err) + } } if len(receiptTxHash) > 0 { @@ -558,17 +564,20 @@ func (a *Application) Commit() abci.ResponseCommit { } a.childTxRefs = nil - go func(height int64, blockHeader abci.Header) { - if err := a.EventHandler.EmitBlockTx(uint64(height), blockHeader.Time); err != nil { - log.Error("Emit Block Event error", "err", err) - } - if err := a.EventHandler.LegacyEthSubscriptionSet().EmitBlockEvent(blockHeader); err != nil { - log.Error("Emit Block Event error", "err", err) - } - if err := a.EventHandler.EthSubscriptionSet().EmitBlockEvent(blockHeader); err != nil { - log.Error("Emit Block Event error", "err", err) - } - }(height, a.curBlockHeader) + if a.EventHandler != nil { + go func(height int64, blockHeader abci.Header) { + if err := a.EventHandler.EmitBlockTx(uint64(height), blockHeader.Time); err != nil { + log.Error("Emit Block Event error", "err", err) + } + if err := a.EventHandler.LegacyEthSubscriptionSet().EmitBlockEvent(blockHeader); err != nil { + log.Error("Emit Block Event error", "err", err) + } + if err := a.EventHandler.EthSubscriptionSet().EmitBlockEvent(blockHeader); err != nil { + log.Error("Emit Block Event error", "err", err) + } + }(height, a.curBlockHeader) + } + a.lastBlockHeader = a.curBlockHeader if err := a.Store.Prune(); err != nil { diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index a43d90a16d..1e143d9427 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -23,8 +23,14 @@ func TraceTransaction( blockstore store.BlockStore, startBlockNumber, targetBlockNumber, txIndex int64, config eth.TraceConfig, -) (interface{}, error) { - if err := runUpTo(app, blockstore, startBlockNumber, targetBlockNumber, txIndex); err != nil { +) (trace interface{}, err error) { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("loomchain panicked %v", r) + } + }() + + if err := runUpTo(&app, blockstore, startBlockNumber, targetBlockNumber, txIndex); err != nil { return nil, err } @@ -56,7 +62,7 @@ func TraceTransaction( } } -func runUpTo(app loomchain.Application, blockstore store.BlockStore, startHeight, height, index int64) error { +func runUpTo(app *loomchain.Application, blockstore store.BlockStore, startHeight, height, index int64) error { for h := startHeight; h <= height; h++ { resultBlock, err := blockstore.GetBlockByHeight(&h) if err != nil { diff --git a/store/splitstore.go b/store/splitstore.go index 5540a178f4..7460e20bea 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -20,7 +20,7 @@ func NewSplitStore(full KVReader, empty VersionedKVStore, version int64) Version } } -func (ss splitStore) Get(key []byte) []byte { +func (ss *splitStore) Get(key []byte) []byte { if ss.VersionedKVStore.Has(key) { return ss.KVReader.Get(key) } @@ -30,7 +30,7 @@ func (ss splitStore) Get(key []byte) []byte { return ss.KVReader.Get(key) } -func (ss splitStore) Range(prefix []byte) plugin.RangeData { +func (ss *splitStore) Range(prefix []byte) plugin.RangeData { readerRange := ss.KVReader.Range(prefix) updateRange := ss.VersionedKVStore.Range(prefix) for _, re := range updateRange { @@ -41,7 +41,7 @@ func (ss splitStore) Range(prefix []byte) plugin.RangeData { return readerRange } -func (ss splitStore) Has(key []byte) bool { +func (ss *splitStore) Has(key []byte) bool { if ss.VersionedKVStore.Has(key) { return true } @@ -51,36 +51,36 @@ func (ss splitStore) Has(key []byte) bool { return ss.KVReader.Has(key) } -func (ss splitStore) Set(key, value []byte) { +func (ss *splitStore) Set(key, value []byte) { ss.VersionedKVStore.Set(key, value) ss.deleted[string(key)] = false } -func (ss splitStore) Delete(key []byte) { +func (ss *splitStore) Delete(key []byte) { ss.VersionedKVStore.Delete(key) ss.deleted[string(key)] = true } -func (ss splitStore) Hash() []byte { +func (ss *splitStore) Hash() []byte { return nil } -func (ss splitStore) Version() int64 { +func (ss *splitStore) Version() int64 { return ss.version } -func (ss splitStore) SaveVersion() ([]byte, int64, error) { +func (ss *splitStore) SaveVersion() ([]byte, int64, error) { ss.version++ return nil, ss.version, nil } -func (ss splitStore) Prune() error { - panic("should not be called") +func (ss *splitStore) Prune() error { + return nil } func (ss *splitStore) GetSnapshot() Snapshot { - return splitStoreSnapShot{*ss} + return &splitStoreSnapShot{*ss} } -func (ss splitStore) GetSnapshotAt(version int64) (Snapshot, error) { +func (ss *splitStore) GetSnapshotAt(version int64) (Snapshot, error) { panic("should not be called") } @@ -88,4 +88,4 @@ type splitStoreSnapShot struct { splitStore } -func (ss splitStoreSnapShot) Release() {} +func (ss *splitStoreSnapShot) Release() {} From de3313fd4c9248d577691923677ea7d35dcd30c7 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 13:59:14 +0000 Subject: [PATCH 32/49] fix version number --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 5ede30ae00..74aac199d4 100644 --- a/app.go +++ b/app.go @@ -623,7 +623,7 @@ func (a *Application) ReadOnlyState() appstate.State { } func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.BlockStore) (*Application, int64, error) { - startVersion := int64(blockNumber) + startVersion := int64(blockNumber) - 1 if startVersion < 0 { return nil, 0, errors.Errorf("invalid block number %d", blockNumber) } From 9c3daa1106c98358c905e70de5d9f2165704ac21 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 16:40:47 +0000 Subject: [PATCH 33/49] copy factory, truffle test --- app.go | 9 ++- e2e/tests/truffle/test/DebugFunctions.js | 81 ++---------------------- txhandler/factory/handler_factory.go | 10 +++ txhandler/txhandler.go | 2 + 4 files changed, 25 insertions(+), 77 deletions(-) diff --git a/app.go b/app.go index 74aac199d4..75de4cfc89 100644 --- a/app.go +++ b/app.go @@ -635,17 +635,19 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo return nil, 0, errors.Errorf("no saved version for height %d", blockNumber) } - txHandle, err := a.TxHandlerFactory.TxHandler(nil, false) + splitStore := store.NewSplitStore(snapshot, store.NewMemStore(), startVersion-1) + factory := a.TxHandlerFactory.Copy(splitStore) + txHandle, err := factory.TxHandler(nil, false) if err != nil { return nil, 0, err } newApp := &Application{ - Store: store.NewSplitStore(snapshot, store.NewMemStore(), startVersion-1), + Store: splitStore, Init: func(state appstate.State) error { panic("init should not be called") }, TxHandler: txHandle, - TxHandlerFactory: a.TxHandlerFactory, + TxHandlerFactory: factory, BlockIndexStore: nil, EventHandler: nil, ReceiptHandlerProvider: nil, @@ -656,6 +658,7 @@ func (a *Application) ReplayApplication(blockNumber uint64, blockstore store.Blo GetValidatorSet: a.GetValidatorSet, EvmAuxStore: nil, ReceiptsVersion: a.ReceiptsVersion, + config: a.config, } return newApp, startVersion, nil } diff --git a/e2e/tests/truffle/test/DebugFunctions.js b/e2e/tests/truffle/test/DebugFunctions.js index a8dae77da3..4bdb04903e 100644 --- a/e2e/tests/truffle/test/DebugFunctions.js +++ b/e2e/tests/truffle/test/DebugFunctions.js @@ -2,20 +2,16 @@ const Web3 = require('web3'); const fs = require('fs'); const path = require('path'); const EthereumTx = require('ethereumjs-tx').Transaction; -const { getLoomEvmTxHash } = require('./helpers'); - +const { getLoomEvmTxHash, waitForXBlocks } = require('./helpers'); const { SpeculativeNonceTxMiddleware, SignedTxMiddleware, Client, LocalAddress, CryptoUtils, LoomProvider } = require('loom-js'); - const TxHashTestContract = artifacts.require('TxHashTestContract'); -// Requires receipts:v3.3 to be enabled, and receipts:v3.4 not to be, but the new tx hash algo needs -// more review & testing before we can release it so skipping this test for now. -contract('TxHashTestContract', async (accounts) => { - let contract, nonceContract, fromAddr, nodeAddr, txHashTestContract, nonceTestContract, web3, web3js; - +contract('debug_traceTransaction', async (accounts) => { + let contract, fromAddr, nodeAddr, txHashTestContract, web3, web3js; + const StructLogsLength = 87; beforeEach(async () => { nodeAddr = fs.readFileSync(path.join(process.env.CLUSTER_DIR, '0', 'node_rpc_addr'), 'utf-8').trim() const chainID = 'default'; @@ -47,80 +43,17 @@ contract('TxHashTestContract', async (accounts) => { }); it('Test debug_traceTransaction', async () => { - console.log("piers contract.methods|", contract.methods); - console.log("piers contract.method.set|", contract.methods.set); - try { - const txResult = await contract.methods.set(1111).send(); - //console.log("piers txResult.tx|", txResult.tx); - // console.log("piers txResult.transactionHash|", txResult.transactionHash); - - const receipt = await web3js.eth.getTransactionReceipt(txResult.transactionHash); - console.log("piers receipt|", receipt); - - let blockNumber = await web3js.eth.getBlockNumber(); - - console.log("blocknumerb before", blockNumber); - //let bn = blockNumber; - //while ( bn <= blockNumber+3) { - // //await contract.methods.set(1111).send(); - // bn = await web3js.eth.getBlockNumber(); - //console.log("Inloop bn", bn); - //} - await waitForXBlocks(nodeAddr, 5) - blockNumber = await web3js.eth.getBlockNumber(); - console.log("blocknumerb after", blockNumber); - - const waitResult = await web3js.currentProvider.send({ + await web3js.currentProvider.send({ method: "debug_traceTransaction", params: [txResult.transactionHash,{"disableStorage":true,"disableMemory":false,"disableStack":false,"fullStorage":false}], jsonrpc: "2.0", id: new Date().getTime() }, function (error, result) { - console.log("piers!!!!!!!!! debug_traceTransaction sendResult|", result, "error", error) - console.log("piers failed|", result.result.failed); - console.log("piers structLogs|", result.result.structLogs); - //assert.equal(true, result === result); - //assert.equal(undefined, result.failed) - assert.equal(true, false); - //let aaa = result.doesnotexist.doesnotexist; + assert.equal(StructLogsLength, result.result.structLogs.length); }); - console.log("waitResult", waitResult); + await waitForXBlocks(nodeAddr, 1) - const sendResult = await web3js.currentProvider.send({ - method: "eth_blockNumber", - params: [], - jsonrpc: "2.0", - id: new Date().getTime() - }, function (error, result) { - console.log("send eth_blockNumber|", result, "error", error) - }); - //console.log("sendResult", sendResult); - - await waitForXBlocks(nodeAddr, 5) - - //sleep1(100000000) - } catch(err) { - console.log("caught error", err); - } - //const dttResult = await web3js.currentProvider.send('debug_traceTransaction', [ txResult.transactionHash ]).then((result) => { - // console.log(result); - //}); - //console.log("dttResult", dttResult) }) }); - -function sleep1(milliseconds) { - let timeStart = new Date().getTime(); - while (true) { - let elapsedTime = new Date().getTime() - timeStart; - if (elapsedTime > milliseconds) { - break; - } - } -} - -const sleep = (milliseconds) => { - return new Promise(resolve => setTimeout(resolve, milliseconds)) -} \ No newline at end of file diff --git a/txhandler/factory/handler_factory.go b/txhandler/factory/handler_factory.go index 75f5bea5ca..12809b5a0f 100644 --- a/txhandler/factory/handler_factory.go +++ b/txhandler/factory/handler_factory.go @@ -46,6 +46,16 @@ type txHandleFactory struct { createRegistry factory.RegistryFactoryFunc } +func (f txHandleFactory) Copy(newStore store.VersionedKVStore) txhandler.TxHandlerFactory { + return txHandleFactory{ + cfg: f.cfg, + vmManager: f.vmManager, + chainID: f.chainID, + store: newStore, + createRegistry: f.createRegistry, + } +} + func (f txHandleFactory) TxHandler(tracer ethvm.Tracer, metrics bool) (txhandler.TxHandler, error) { vmManager := createVmManager(f.vmManager, tracer) nonceTxHandler := auth.NewNonceHandler() diff --git a/txhandler/txhandler.go b/txhandler/txhandler.go index 28e85d30a2..ec2033e7d9 100644 --- a/txhandler/txhandler.go +++ b/txhandler/txhandler.go @@ -6,6 +6,7 @@ import ( "github.com/tendermint/tendermint/libs/common" "github.com/loomnetwork/loomchain/state" + "github.com/loomnetwork/loomchain/store" ) type TxHandler interface { @@ -29,4 +30,5 @@ func (f TxHandlerFunc) ProcessTx(s state.State, txBytes []byte, isCheckTx bool) type TxHandlerFactory interface { TxHandler(tracer vm.Tracer, metrics bool) (TxHandler, error) + Copy(newStore store.VersionedKVStore) TxHandlerFactory } From ba19cf4afd72aac43c279b92c51c59faba50e4d1 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 16:56:12 +0000 Subject: [PATCH 34/49] revert --- e2e/tests/truffle/test/DebugFunctions.js | 2 - store/iavlstore.go | 51 ++++++------------------ 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/e2e/tests/truffle/test/DebugFunctions.js b/e2e/tests/truffle/test/DebugFunctions.js index 4bdb04903e..05d78a9d44 100644 --- a/e2e/tests/truffle/test/DebugFunctions.js +++ b/e2e/tests/truffle/test/DebugFunctions.js @@ -53,7 +53,5 @@ contract('debug_traceTransaction', async (accounts) => { assert.equal(StructLogsLength, result.result.structLogs.length); }); await waitForXBlocks(nodeAddr, 1) - }) - }); diff --git a/store/iavlstore.go b/store/iavlstore.go index 7ea1ecb5a2..df15af1e77 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -1,7 +1,6 @@ package store import ( - "bytes" "fmt" "time" @@ -9,11 +8,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/util" - "github.com/loomnetwork/loomchain/log" "github.com/pkg/errors" stdprometheus "github.com/prometheus/client_golang/prometheus" "github.com/tendermint/iavl" dbm "github.com/tendermint/tendermint/libs/db" + + "github.com/loomnetwork/loomchain/log" ) var ( @@ -211,15 +211,17 @@ func (s *IAVLStore) Prune() error { } func (s *IAVLStore) GetSnapshot() Snapshot { - snapshot, err := s.GetSnapshotAt(0) - if err != nil { - panic(err) + // This isn't an actual snapshot obviously, and never will be, but lets pretend... + return &iavlStoreSnapshot{ + IAVLStore: s, } - return snapshot } func (s *IAVLStore) GetSnapshotAt(version int64) (Snapshot, error) { - return newIavlStoreSnapshot(*s, version) + // This isn't an actual snapshot obviously, and never will be, but lets pretend... + return &iavlStoreSnapshot{ + IAVLStore: s, + }, nil } // NewIAVLStore creates a new IAVLStore. @@ -255,38 +257,9 @@ func NewIAVLStore(db dbm.DB, maxVersions, targetVersion, flushInterval int64) (* } type iavlStoreSnapshot struct { - *iavl.ImmutableTree + *IAVLStore } -func newIavlStoreSnapshot(store IAVLStore, version int64) (Snapshot, error) { - if version == 0 { - version = store.Version() - } - immutableTree, err := store.tree.GetImmutable(version) - if err != nil { - return nil, err - } - return &iavlStoreSnapshot{ - immutableTree, - }, nil -} - -func (s *iavlStoreSnapshot) Get(key []byte) []byte { - _, value := s.ImmutableTree.Get(key) - return value +func (s *iavlStoreSnapshot) Release() { + // noop } - -func (s *iavlStoreSnapshot) Range(prefix []byte) plugin.RangeData { - var data plugin.RangeData - prefix = append(prefix, 0) - s.ImmutableTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { - if len(key) < len(prefix) || 0 != bytes.Compare(prefix, key[:len(prefix)]) { - return true - } - data = append(data, &plugin.RangeEntry{key, value}) - return false - }) - return data -} - -func (s *iavlStoreSnapshot) Release() {} From 323fa14ed407cdc4a79cdb0c0bfe0c30dea4cdc5 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 31 Oct 2019 17:21:09 +0000 Subject: [PATCH 35/49] unit test fixes --- auth/multi_chain_sigtx_middleware_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/auth/multi_chain_sigtx_middleware_test.go b/auth/multi_chain_sigtx_middleware_test.go index b7f37e4708..e9bacdc98b 100644 --- a/auth/multi_chain_sigtx_middleware_test.go +++ b/auth/multi_chain_sigtx_middleware_test.go @@ -35,7 +35,6 @@ import ( ) const ( - sequence = uint64(4) callId = uint32(2) seq = uint64(4) // todo defaultLoomChainId = "default" From 8f3a7ad75981b957dc67e03b15f92c01042d2211 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 1 Nov 2019 11:06:56 +0000 Subject: [PATCH 36/49] unit test fixes --- cmd/loom/handler_test.go | 13 +- eth/polls/polls_test.go | 10 +- eth/query/query_test.go | 2 +- rpc/debug/noevmtrace.go | 19 ++ rpc/json_handler_test.go | 4 +- rpc/query_server_test.go | 31 +-- store/splitstore.go | 6 +- store/store_test.go | 2 +- throttle/contract_tx_limiter_middleware.go | 241 ------------------ throttle/tx_limiter_middleware.go | 1 - .../contract_tx_limiter_middleware_test.go | 10 +- .../middleware/deployer-middleware_test.go | 16 +- txhandler/middleware/karma-middleware_test.go | 29 ++- .../middleware/middleware_test_helper.go | 2 +- .../middleware}/throttle_test.go | 24 +- .../migration/migration_tx_handler_test.go | 4 +- 16 files changed, 95 insertions(+), 319 deletions(-) create mode 100644 rpc/debug/noevmtrace.go delete mode 100644 throttle/contract_tx_limiter_middleware.go delete mode 100644 throttle/tx_limiter_middleware.go rename {throttle => txhandler/middleware}/contract_tx_limiter_middleware_test.go (98%) rename {throttle => txhandler/middleware}/throttle_test.go (93%) diff --git a/cmd/loom/handler_test.go b/cmd/loom/handler_test.go index 03947e75a9..2db34359ab 100644 --- a/cmd/loom/handler_test.go +++ b/cmd/loom/handler_test.go @@ -12,11 +12,12 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "golang.org/x/crypto/ed25519" - "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/auth" registry "github.com/loomnetwork/loomchain/registry/factory" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/txhandler" + "github.com/loomnetwork/loomchain/txhandler/middleware" "github.com/loomnetwork/loomchain/vm" ) @@ -32,19 +33,19 @@ func TestTxHandlerWithInvalidCaller(t *testing.T) { require.NoError(t, err) vmManager := vm.NewManager() - router := loomchain.NewTxRouter() - router.HandleDeliverTx(1, loomchain.GeneratePassthroughRouteHandler(&vm.DeployTxHandler{Manager: vmManager, CreateRegistry: createRegistry})) - router.HandleDeliverTx(2, loomchain.GeneratePassthroughRouteHandler(&vm.CallTxHandler{Manager: vmManager})) + router := middleware.NewTxRouter() + router.HandleDeliverTx(1, middleware.GeneratePassthroughRouteHandler(&vm.DeployTxHandler{Manager: vmManager, CreateRegistry: createRegistry})) + router.HandleDeliverTx(2, middleware.GeneratePassthroughRouteHandler(&vm.CallTxHandler{Manager: vmManager})) kvStore := store.NewMemStore() state := appstate.NewStoreState(nil, kvStore, abci.Header{ChainID: "default"}, nil, nil) - txMiddleWare := []loomchain.TxMiddleware{ + txMiddleWare := []txhandler.TxMiddleware{ auth.SignatureTxMiddleware, auth.NewNonceHandler().TxMiddleware(kvStore), } - rootHandler := loomchain.MiddlewareTxHandler(txMiddleWare, router, nil) + rootHandler := txhandler.MiddlewareTxHandler(txMiddleWare, router, nil) signer := lauth.NewEd25519Signer(alicePrivKey) caller := loom.Address{ ChainID: "default", diff --git a/eth/polls/polls_test.go b/eth/polls/polls_test.go index 689c552283..75e4a31ba6 100644 --- a/eth/polls/polls_test.go +++ b/eth/polls/polls_test.go @@ -9,18 +9,18 @@ import ( "github.com/loomnetwork/loomchain/auth" "github.com/loomnetwork/loomchain/rpc/eth" - "github.com/loomnetwork/loomchain/vm" appstate "github.com/loomnetwork/loomchain/state" - - "github.com/loomnetwork/loomchain/events" - "github.com/loomnetwork/loomchain/store" + "github.com/loomnetwork/loomchain/vm" "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom" "github.com/loomnetwork/go-loom/plugin/types" + ltypes "github.com/loomnetwork/go-loom/types" "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/events" "github.com/loomnetwork/loomchain/receipts/common" "github.com/loomnetwork/loomchain/receipts/handler" + "github.com/loomnetwork/loomchain/store" "github.com/stretchr/testify/require" ) @@ -397,7 +397,7 @@ func mockSignedTx(t *testing.T, id uint32, to loom.Address, from loom.Address, d }) require.NoError(t, err) - txTx, err := proto.Marshal(&loomchain.Transaction{ + txTx, err := proto.Marshal(<ypes.Transaction{ Data: messageTx, Id: id, }) diff --git a/eth/query/query_test.go b/eth/query/query_test.go index 1269cd6c6a..245ab3d062 100644 --- a/eth/query/query_test.go +++ b/eth/query/query_test.go @@ -357,7 +357,7 @@ func mockSignedTx(t *testing.T, id ltypes.TxID, to loom.Address, from loom.Addre }) require.NoError(t, err) - txTx, err := proto.Marshal(&loomchain.Transaction{ + txTx, err := proto.Marshal(<ypes.Transaction{ Data: messageTx, Id: uint32(id), }) diff --git a/rpc/debug/noevmtrace.go b/rpc/debug/noevmtrace.go new file mode 100644 index 0000000000..5781d01c97 --- /dev/null +++ b/rpc/debug/noevmtrace.go @@ -0,0 +1,19 @@ +// +build !evm + +package debug + +import ( + "github.com/ethereum/go-ethereum/eth" + + "github.com/loomnetwork/loomchain" + "github.com/loomnetwork/loomchain/store" +) + +func TraceTransaction( + _ loomchain.Application, + _ store.BlockStore, + _, _, _ int64, + _ eth.TraceConfig, +) (trace interface{}, err error) { + return nil, nil +} diff --git a/rpc/json_handler_test.go b/rpc/json_handler_test.go index 6944abc88a..42ef72c595 100644 --- a/rpc/json_handler_test.go +++ b/rpc/json_handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/events" "github.com/loomnetwork/loomchain/log" "github.com/loomnetwork/loomchain/registry/factory" @@ -116,7 +116,7 @@ func testEthSubscribeEthUnSubscribe(t *testing.T) { Loader: loader, CreateRegistry: createRegistry, BlockStore: store.NewMockBlockStore(), - AuthCfg: auth.DefaultConfig(), + AuthCfg: keys.DefaultConfig(), EthSubscriptions: eventHandler.EthSubscriptionSet(), } handler := MakeEthQueryServiceHandler(testlog, hub, createDefaultEthRoutes(qs, "default")) diff --git a/rpc/query_server_test.go b/rpc/query_server_test.go index cccb311245..049ad74089 100644 --- a/rpc/query_server_test.go +++ b/rpc/query_server_test.go @@ -13,19 +13,20 @@ import ( proto "github.com/gogo/protobuf/proto" lp "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/types" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + dbm "github.com/tendermint/tendermint/libs/db" + rpcclient "github.com/tendermint/tendermint/rpc/lib/client" + "github.com/loomnetwork/loomchain" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/eth/subs" llog "github.com/loomnetwork/loomchain/log" "github.com/loomnetwork/loomchain/plugin" registry "github.com/loomnetwork/loomchain/registry/factory" "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" - stdprometheus "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" - dbm "github.com/tendermint/tendermint/libs/db" - rpcclient "github.com/tendermint/tendermint/rpc/lib/client" ) type queryableContract struct { @@ -102,16 +103,8 @@ func (s *stateProvider) ReadOnlyState() state.State { ) } -func (s *stateProvider) InMemoryApp() state.State { - return state.NewStoreState( - nil, - store.NewMemStore(), - abci.Header{ - ChainID: s.ChainID, - }, - nil, - nil, - ) +func (s *stateProvider) ReplayApplication(uint64, store.BlockStore) (*loomchain.Application, int64, error) { + return nil, 0, fmt.Errorf("Not implimented") } var testlog llog.TMLogger @@ -136,7 +129,7 @@ func testQueryServerContractQuery(t *testing.T) { Loader: loader, CreateRegistry: createRegistry, BlockStore: store.NewMockBlockStore(), - AuthCfg: auth.DefaultConfig(), + AuthCfg: keys.DefaultConfig(), } bus := &QueryEventBus{ Subs: *loomchain.NewSubscriptionSet(), @@ -191,7 +184,7 @@ func testQueryServerNonce(t *testing.T) { ChainID: "default", }, BlockStore: store.NewMockBlockStore(), - AuthCfg: auth.DefaultConfig(), + AuthCfg: keys.DefaultConfig(), } bus := &QueryEventBus{ Subs: *loomchain.NewSubscriptionSet(), @@ -260,7 +253,7 @@ func testQueryMetric(t *testing.T) { Loader: loader, CreateRegistry: createRegistry, BlockStore: store.NewMockBlockStore(), - AuthCfg: auth.DefaultConfig(), + AuthCfg: keys.DefaultConfig(), } qs = InstrumentingMiddleware{requestCount, requestLatency, qs} bus := &QueryEventBus{ diff --git a/store/splitstore.go b/store/splitstore.go index 7460e20bea..493ed643cc 100644 --- a/store/splitstore.go +++ b/store/splitstore.go @@ -62,14 +62,16 @@ func (ss *splitStore) Delete(key []byte) { } func (ss *splitStore) Hash() []byte { - return nil + return []byte{} } + func (ss *splitStore) Version() int64 { return ss.version } + func (ss *splitStore) SaveVersion() ([]byte, int64, error) { ss.version++ - return nil, ss.version, nil + return ss.Hash(), ss.version, nil } func (ss *splitStore) Prune() error { diff --git a/store/store_test.go b/store/store_test.go index 40065bff2e..1d51792ed5 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -450,7 +450,7 @@ type SplitStoreTestSuite struct { // runs before each test in this suite func (ts *SplitStoreTestSuite) SetupTest() { - ts.store = NewSplitStore(NewMemStore(), NewMemStore()) + ts.store = NewSplitStore(NewMemStore(), NewMemStore(), 1) } func (ts *SplitStoreTestSuite) SetupSuite() { diff --git a/throttle/contract_tx_limiter_middleware.go b/throttle/contract_tx_limiter_middleware.go deleted file mode 100644 index 7b696d1336..0000000000 --- a/throttle/contract_tx_limiter_middleware.go +++ /dev/null @@ -1,241 +0,0 @@ -package throttle - -import ( - "fmt" - "time" - - "github.com/go-kit/kit/metrics" - "github.com/gogo/protobuf/proto" - "github.com/loomnetwork/go-loom" - udwtypes "github.com/loomnetwork/go-loom/builtin/types/user_deployer_whitelist" - "github.com/loomnetwork/go-loom/plugin/contractpb" - ltypes "github.com/loomnetwork/go-loom/types" - "github.com/pkg/errors" - - "github.com/loomnetwork/loomchain/auth" - udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" - appstate "github.com/loomnetwork/loomchain/state" - "github.com/loomnetwork/loomchain/txhandler" - "github.com/loomnetwork/loomchain/vm" -) - -var ( - ErrTxLimitReached = errors.New("tx limit reached, try again later") - ErrContractNotWhitelisted = errors.New("contract not whitelisted") - ErrInactiveDeployer = errors.New("can't call contract belonging to inactive deployer") -) - -var ( - tierMapLoadLatency metrics.Histogram - contractTierMapLoadLatency metrics.Histogram -) - -type ContractTxLimiterConfig struct { - // Enables the middleware - Enabled bool - // Number of seconds each refresh lasts - ContractDataRefreshInterval int64 - TierDataRefreshInterval int64 -} - -func DefaultContractTxLimiterConfig() *ContractTxLimiterConfig { - return &ContractTxLimiterConfig{ - Enabled: false, - ContractDataRefreshInterval: 15 * 60, - TierDataRefreshInterval: 15 * 60, - } -} - -// Clone returns a deep clone of the config. -func (c *ContractTxLimiterConfig) Clone() *ContractTxLimiterConfig { - if c == nil { - return nil - } - clone := *c - return &clone -} - -type contractTxLimiter struct { - // contract_address to limiting parametres structure - contractToTierMap map[string]udw.TierID - inactiveDeployerContracts map[string]bool - contractDataLastUpdated int64 - // track of no. of txns in previous blocks per contract - contractStatsMap map[string]*contractStats - tierMap map[udw.TierID]udw.Tier - tierDataLastUpdated int64 -} - -type contractStats struct { - txn int64 - blockHeight int64 -} - -func (txl *contractTxLimiter) isAccountLimitReached(contractAddr loom.Address, curBlockHeight int64) bool { - blockTx, ok := txl.contractStatsMap[contractAddr.String()] - if !ok { - return false - } - // if execution reaches here => tierID and tier are valid - tierID := txl.contractToTierMap[contractAddr.String()] - tier := txl.tierMap[tierID] - if blockTx.blockHeight <= (curBlockHeight-int64(tier.BlockRange)) || int64(tier.MaxTxs) > blockTx.txn { - return false - } - return true -} - -func (txl *contractTxLimiter) updateState(contractAddr loom.Address, curBlockHeight int64) { - blockTx, ok := txl.contractStatsMap[contractAddr.String()] - tierID := txl.contractToTierMap[contractAddr.String()] - tier := txl.tierMap[tierID] - blockRange := int64(4096) // prevent divide by zero just in case tier doesn't have a range set - if tier.BlockRange > 0 { - blockRange = int64(tier.BlockRange) - } - if !ok || blockTx.blockHeight <= (curBlockHeight-blockRange) { - // resetting the blockHeight to lower bound of range instead of curblockheight - rangeStart := (((curBlockHeight - 1) / blockRange) * blockRange) + 1 - txl.contractStatsMap[contractAddr.String()] = &contractStats{1, rangeStart} - return - } - blockTx.txn++ -} - -func loadContractTierMap(ctx contractpb.StaticContext) (*udw.ContractInfo, error) { - var err error - defer func(begin time.Time) { - lvs := []string{"method", "loadContractTierMap", "error", fmt.Sprint(err != nil)} - contractTierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) - }(time.Now()) - contractInfo, err := udw.GetContractInfo(ctx) - return contractInfo, err -} - -func loadTierMap(ctx contractpb.StaticContext) (map[udwtypes.TierID]udwtypes.Tier, error) { - var err error - defer func(begin time.Time) { - lvs := []string{"method", "loadTierMap", "error", fmt.Sprint(err != nil)} - tierMapLoadLatency.With(lvs...).Observe(time.Since(begin).Seconds()) - }(time.Now()) - tierMap, err := udw.GetTierMap(ctx) - return tierMap, err -} - -// NewContractTxLimiterMiddleware creates a middleware function that limits how many call txs can be -// sent to an EVM contract within a pre-configured block range. -func NewContractTxLimiterMiddleware(cfg *ContractTxLimiterConfig, - createUserDeployerWhitelistCtx func(state appstate.State) (contractpb.Context, error), -) txhandler.TxMiddlewareFunc { - txl := &contractTxLimiter{ - contractStatsMap: make(map[string]*contractStats), - } - return txhandler.TxMiddlewareFunc(func( - state appstate.State, - txBytes []byte, - next txhandler.TxHandlerFunc, - isCheckTx bool, - ) (res txhandler.TxHandlerResult, err error) { - if !isCheckTx { - return next(state, txBytes, isCheckTx) - } - var nonceTx auth.NonceTx - if err := proto.Unmarshal(txBytes, &nonceTx); err != nil { - return res, errors.Wrap(err, "throttle: unwrap nonce Tx") - } - var tx ltypes.Transaction - if err := proto.Unmarshal(nonceTx.Inner, &tx); err != nil { - return res, errors.New("throttle: unmarshal tx") - } - - var msg vm.MessageTx - switch ltypes.TxID(tx.Id) { - case ltypes.TxID_CALL: - if err := proto.Unmarshal(tx.Data, &msg); err != nil { - return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) - } - var callTx vm.CallTx - if err := proto.Unmarshal(msg.Data, &callTx); err != nil { - return res, errors.Wrapf(err, "unmarshal call tx %v", msg.Data) - } - if callTx.VmType != vm.VMType_EVM { - return next(state, txBytes, isCheckTx) - } - - case ltypes.TxID_ETHEREUM: - if err := proto.Unmarshal(tx.Data, &msg); err != nil { - return res, errors.Wrapf(err, "unmarshal message tx %v", tx.Data) - } - isDeploy, err := IsEthDeploy(msg.Data) - if err != nil { - return res, err - } - if isDeploy { - return next(state, txBytes, isCheckTx) - } - - default: - return next(state, txBytes, isCheckTx) - } - - if txl.inactiveDeployerContracts == nil || - txl.contractToTierMap == nil || - (txl.contractDataLastUpdated+cfg.ContractDataRefreshInterval) < time.Now().Unix() { - ctx, err := createUserDeployerWhitelistCtx(state) - if err != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - contractInfo, err := loadContractTierMap(ctx) - if err != nil { - return res, errors.Wrap(err, "throttle: contractInfo fetch") - } - - txl.contractDataLastUpdated = time.Now().Unix() - txl.contractToTierMap = contractInfo.ContractToTierMap - txl.inactiveDeployerContracts = contractInfo.InactiveDeployerContracts - // TxLimiter.contractDataLastUpdated will be updated after updating contractToTierMap - } - contractAddr := loom.UnmarshalAddressPB(msg.To) - // contracts which are deployed by deleted deployers should be throttled - if txl.inactiveDeployerContracts[contractAddr.String()] { - return res, ErrInactiveDeployer - } - // contracts the limiter doesn't know about shouldn't be throttled - contractTierID, ok := txl.contractToTierMap[contractAddr.String()] - if !ok { - return next(state, txBytes, isCheckTx) - } - if txl.tierMap == nil || - (txl.tierDataLastUpdated+cfg.TierDataRefreshInterval) < time.Now().Unix() { - ctx, er := createUserDeployerWhitelistCtx(state) - if er != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - txl.tierMap, err = loadTierMap(ctx) - if err != nil { - return res, errors.Wrap(err, "throttle: GetTierMap error") - } - txl.tierDataLastUpdated = time.Now().Unix() - } - // ensure that tier corresponding to contract available in tierMap - _, ok = txl.tierMap[contractTierID] - if !ok { - ctx, er := createUserDeployerWhitelistCtx(state) - if er != nil { - return res, errors.Wrap(err, "throttle: context creation") - } - tierInfo, er := udw.GetTierInfo(ctx, contractTierID) - if er != nil { - return res, errors.Wrap(err, "throttle: getTierInfo error") - } - txl.tierMap[contractTierID] = tierInfo - } - - if txl.isAccountLimitReached(contractAddr, state.Block().Height) { - return txhandler.TxHandlerResult{}, ErrTxLimitReached - } - txl.updateState(contractAddr, state.Block().Height) - - return next(state, txBytes, isCheckTx) - }) -} diff --git a/throttle/tx_limiter_middleware.go b/throttle/tx_limiter_middleware.go deleted file mode 100644 index 46736e514d..0000000000 --- a/throttle/tx_limiter_middleware.go +++ /dev/null @@ -1 +0,0 @@ -package throttle diff --git a/throttle/contract_tx_limiter_middleware_test.go b/txhandler/middleware/contract_tx_limiter_middleware_test.go similarity index 98% rename from throttle/contract_tx_limiter_middleware_test.go rename to txhandler/middleware/contract_tx_limiter_middleware_test.go index 42b7b5b376..451e314337 100644 --- a/throttle/contract_tx_limiter_middleware_test.go +++ b/txhandler/middleware/contract_tx_limiter_middleware_test.go @@ -1,6 +1,6 @@ // +build evm -package throttle +package middleware import ( "testing" @@ -11,16 +11,18 @@ import ( goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/loomnetwork/loomchain/builtin/plugins/coin" "github.com/loomnetwork/loomchain/builtin/plugins/deployer_whitelist" udw "github.com/loomnetwork/loomchain/builtin/plugins/user_deployer_whitelist" + "github.com/loomnetwork/loomchain/config" "github.com/loomnetwork/loomchain/features" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" ) var ( @@ -103,7 +105,7 @@ func TestContractTxLimiterMiddleware(t *testing.T) { state := appstate.NewStoreState(nil, store.NewMemStore(), abci.Header{Height: 5}, nil, nil) //EVMTxn txSignedEVM1 := mockSignedTx(t, uint64(1), types.TxID_CALL, vm.VMType_EVM, contractAddr) - cfg := DefaultContractTxLimiterConfig() + cfg := config.DefaultContractTxLimiterConfig() contractTxLimiterMiddleware := NewContractTxLimiterMiddleware(cfg, func(_ appstate.State) (contractpb.Context, error) { return contractpb.WrapPluginContext(udwContext), nil diff --git a/txhandler/middleware/deployer-middleware_test.go b/txhandler/middleware/deployer-middleware_test.go index 409986062e..b1108f6c90 100644 --- a/txhandler/middleware/deployer-middleware_test.go +++ b/txhandler/middleware/deployer-middleware_test.go @@ -57,24 +57,24 @@ func TestDeployerWhitelistMiddleware(t *testing.T) { require.NoError(t, err) // unauthorized deployer (DeployTx Plugin) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedPlugin, guestCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedPlugin, guestCtx) require.Equal(t, ErrNotAuthorized, err) // unauthorized deployer (DeployTx EVM) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedEVM, guestCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedEVM, guestCtx) require.Equal(t, ErrNotAuthorized, err) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedEth, guestCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedEth, guestCtx) require.Equal(t, ErrNotAuthorized, err) // unauthorized deployer (MigrationTx) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedMigration, guestCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedMigration, guestCtx) require.Equal(t, ErrNotAuthorized, err) // authorized deployer - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedPlugin, ownerCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedPlugin, ownerCtx) require.NoError(t, err) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedEVM, ownerCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedEVM, ownerCtx) require.NoError(t, err) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedEth, ownerCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedEth, ownerCtx) require.NoError(t, err) - _, err = throttleMiddlewareHandler(dwMiddleware, state, txSignedMigration, ownerCtx) + _, err = ThrottleMiddlewareHandler(dwMiddleware, state, txSignedMigration, ownerCtx) require.NoError(t, err) } diff --git a/txhandler/middleware/karma-middleware_test.go b/txhandler/middleware/karma-middleware_test.go index c912b87191..6d5371297d 100644 --- a/txhandler/middleware/karma-middleware_test.go +++ b/txhandler/middleware/karma-middleware_test.go @@ -11,14 +11,15 @@ import ( goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" - "github.com/loomnetwork/loomchain/auth" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/karma" "github.com/loomnetwork/loomchain/log" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" "github.com/loomnetwork/loomchain/vm" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" ) var ( @@ -56,7 +57,7 @@ func TestKarmaMiddleWare(t *testing.T) { // This can also be done on init, but more concise this way require.NoError(t, karma.AddKarma(contractContext, origin, sourceStatesDeploy)) - ctx := context.WithValue(state.Context(), auth.ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) tmx := GetKarmaMiddleWare( true, @@ -69,23 +70,23 @@ func TestKarmaMiddleWare(t *testing.T) { // call fails as contract is not deployed txSigned := mockSignedTx(t, uint64(1), types.TxID_CALL, vm.VMType_EVM, contract) - _, err := throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err := ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.Error(t, err) txSigned = mockSignedTx(t, uint64(1), types.TxID_ETHEREUM, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.Error(t, err) // deploy contract txSigned = mockSignedTx(t, uint64(2), types.TxID_DEPLOY, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.NoError(t, err) // call now works txSigned = mockSignedTx(t, uint64(3), types.TxID_CALL, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.NoError(t, err) txSigned = mockSignedTx(t, uint64(1), types.TxID_ETHEREUM, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.NoError(t, err) // deactivate contract @@ -95,10 +96,10 @@ func TestKarmaMiddleWare(t *testing.T) { // call now fails txSigned = mockSignedTx(t, uint64(4), types.TxID_CALL, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.Error(t, err) txSigned = mockSignedTx(t, uint64(1), types.TxID_ETHEREUM, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.Error(t, err) } @@ -124,7 +125,7 @@ func TestMinKarmaToDeploy(t *testing.T) { require.NoError(t, karma.AddKarma(contractContext, origin, sourceStatesDeploy)) - ctx := context.WithValue(state.Context(), auth.ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) tmx := GetKarmaMiddleWare( true, @@ -137,7 +138,7 @@ func TestMinKarmaToDeploy(t *testing.T) { // deploy contract txSigned := mockSignedTx(t, uint64(2), types.TxID_DEPLOY, vm.VMType_EVM, contract) - _, err := throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err := ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.NoError(t, err) require.NoError(t, karma.SetConfig(contractContext, &ktypes.KarmaConfig{ @@ -146,6 +147,6 @@ func TestMinKarmaToDeploy(t *testing.T) { // deploy contract txSigned = mockSignedTx(t, uint64(2), types.TxID_DEPLOY, vm.VMType_EVM, contract) - _, err = throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err = ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) require.Error(t, err) } diff --git a/txhandler/middleware/middleware_test_helper.go b/txhandler/middleware/middleware_test_helper.go index f15a7646e4..239d792935 100644 --- a/txhandler/middleware/middleware_test_helper.go +++ b/txhandler/middleware/middleware_test_helper.go @@ -21,7 +21,7 @@ var ( contract = loom.MustParseAddress("chain:0x9a1aC42a17AAD6Dbc6d21c162989d0f701074044") ) -func throttleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, tx auth.SignedTx, ctx context.Context) (txhandler.TxHandlerResult, error) { +func ThrottleMiddlewareHandler(ttm txhandler.TxMiddlewareFunc, state appstate.State, tx auth.SignedTx, ctx context.Context) (txhandler.TxHandlerResult, error) { return ttm.ProcessTx( state.WithContext(ctx), tx.Inner, diff --git a/throttle/throttle_test.go b/txhandler/middleware/throttle_test.go similarity index 93% rename from throttle/throttle_test.go rename to txhandler/middleware/throttle_test.go index 08043f68cd..8f6cb8c9de 100644 --- a/throttle/throttle_test.go +++ b/txhandler/middleware/throttle_test.go @@ -1,6 +1,6 @@ // +build evm -package throttle +package middleware import ( "context" @@ -20,16 +20,16 @@ import ( goloomplugin "github.com/loomnetwork/go-loom/plugin" "github.com/loomnetwork/go-loom/plugin/contractpb" "github.com/loomnetwork/go-loom/types" - loomAuth "github.com/loomnetwork/loomchain/auth" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + "golang.org/x/crypto/ed25519" + + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/builtin/plugins/karma" "github.com/loomnetwork/loomchain/log" appstate "github.com/loomnetwork/loomchain/state" "github.com/loomnetwork/loomchain/store" - "github.com/loomnetwork/loomchain/txhandler" "github.com/loomnetwork/loomchain/vm" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" - "golang.org/x/crypto/ed25519" ) const ( @@ -82,7 +82,7 @@ func TestDeployThrottleTxMiddleware(t *testing.T) { // This can also be done on init, but more concise this way require.NoError(t, karma.AddKarma(contractContext, origin, sourceStates)) - ctx := context.WithValue(state.Context(), loomAuth.ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) tmx := GetKarmaMiddleWare( true, @@ -97,7 +97,7 @@ func TestDeployThrottleTxMiddleware(t *testing.T) { for i := int64(1); i <= deployKarma.Value.Int64()+1; i++ { txSigned := mockSignedTx(t, uint64(i), types.TxID_DEPLOY, vm.VMType_PLUGIN, contract) - _, err := throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err := ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) if i <= deployKarma.Value.Int64() { require.NoError(t, err) @@ -106,7 +106,7 @@ func TestDeployThrottleTxMiddleware(t *testing.T) { for i := int64(1); i <= deployKarma.Value.Int64()+1; i++ { txSigned := mockSignedTx(t, uint64(i), types.TxID_ETHEREUM, vm.VMType_EVM, loom.Address{}) - _, err := throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err := ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) if i <= deployKarma.Value.Int64() { require.NoError(t, err) @@ -133,7 +133,7 @@ func TestCallThrottleTxMiddleware(t *testing.T) { // This can also be done on init, but more concise this way require.NoError(t, karma.AddKarma(contractContext, origin, sourceStates)) - ctx := context.WithValue(state.Context(), loomAuth.ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) tmx := GetKarmaMiddleWare( true, @@ -148,7 +148,7 @@ func TestCallThrottleTxMiddleware(t *testing.T) { for i := int64(1); i <= maxCallCount*2+callKarma.Value.Int64(); i++ { txSigned := mockSignedTx(t, uint64(i), types.TxID_CALL, vm.VMType_PLUGIN, contract) - _, err := throttleMiddlewareHandler(tmx, state, txSigned, ctx) + _, err := ThrottleMiddlewareHandler(tmx, state, txSigned, ctx) if i <= maxCallCount+callKarma.Value.Int64() { require.NoError(t, err) @@ -218,7 +218,7 @@ func mockSignedTx(t *testing.T, sequence uint64, id types.TxID, vmType vm.VMType require.FailNow(t, "invalid tx ID") } - tx, err := proto.Marshal(&txhandler.Transaction{ + tx, err := proto.Marshal(&types.Transaction{ Id: uint32(id), Data: messageTx, }) diff --git a/txhandler/migration/migration_tx_handler_test.go b/txhandler/migration/migration_tx_handler_test.go index 0ceee73951..84896d2a03 100644 --- a/txhandler/migration/migration_tx_handler_test.go +++ b/txhandler/migration/migration_tx_handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/auth/keys" "github.com/loomnetwork/loomchain/features" "github.com/loomnetwork/loomchain/migrations" appstate "github.com/loomnetwork/loomchain/state" @@ -24,7 +24,7 @@ func TestMigrationTxHandler(t *testing.T) { state := appstate.NewStoreState(nil, store.NewMemStore(), abci.Header{}, nil, nil) state.SetFeature(features.MigrationTxFeature, true) - ctx := context.WithValue(state.Context(), auth.ContextKeyOrigin, origin) + ctx := context.WithValue(state.Context(), keys.ContextKeyOrigin, origin) s := state.WithContext(ctx) migrationTx1 := mockMessageTx(t, uint32(1), origin, origin, []byte{}) From ce35dd13ed7f36173bd07815d056b80c2e3a9dcc Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 1 Nov 2019 14:54:56 +0000 Subject: [PATCH 37/49] test jenkins --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ec42c90c2..c27186f6a5 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,8 @@ GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. TG_GIT_REV = config # loomnetwork/go-ethereum loomchain branch -ETHEREUM_GIT_REV = ethapi +#ETHEREUM_GIT_REV = ethapi +ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9 # use go-plugin we get 'timeout waiting for connection info' error HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961 LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9 From 5e10bd1431c9636ecf9c0b550b57fdab4c4f27d6 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 1 Nov 2019 15:18:58 +0000 Subject: [PATCH 38/49] test jenkins --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c27186f6a5..87472eda12 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,8 @@ BINANCE_TGORACLE_DIR=$(GOPATH)/src/$(PKG_BINANCE_TGORACLE) # specific commit. GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. -TG_GIT_REV = config +#TG_GIT_REV = config +TG_GIT_REV = HEAD # loomnetwork/go-ethereum loomchain branch #ETHEREUM_GIT_REV = ethapi ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9 From 37838dfbab6c8c80b7a3f85baef7fa61af2aedd7 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 10:21:49 +0000 Subject: [PATCH 39/49] merge in master changes --- config/config.go | 14 ++++++++++++-- eth/query/noevm.go | 4 ++-- evm/noevm.go | 3 ++- rpc/eth/config.go | 14 -------------- 4 files changed, 16 insertions(+), 19 deletions(-) delete mode 100644 rpc/eth/config.go diff --git a/config/config.go b/config/config.go index 5fd4a4686f..0746460a82 100755 --- a/config/config.go +++ b/config/config.go @@ -20,7 +20,6 @@ import ( hsmpv "github.com/loomnetwork/loomchain/privval/hsm" "github.com/loomnetwork/loomchain/receipts/common" registry "github.com/loomnetwork/loomchain/registry/factory" - "github.com/loomnetwork/loomchain/rpc/eth" "github.com/loomnetwork/loomchain/store" blockindex "github.com/loomnetwork/loomchain/store/block_index" @@ -144,7 +143,7 @@ type Config struct { // Set to true to disable minimum required build number check on node startup SkipMinBuildCheck bool - Web3 *eth.Web3Config + Web3 *Web3Config } type Metrics struct { @@ -212,6 +211,17 @@ type UserDeployerWhitelistConfig struct { ContractEnabled bool } +type Web3Config struct { + // GetLogsMaxBlockRange specifies the maximum number of blocks eth_getLogs will query per request + GetLogsMaxBlockRange uint64 +} + +func DefaultWeb3Config() *Web3Config { + return &Web3Config{ + GetLogsMaxBlockRange: 20, + } +} + func DefaultDBBackendConfig() *DBBackendConfig { return &DBBackendConfig{ CacheSizeMegs: 1042, //1 Gigabyte diff --git a/eth/query/noevm.go b/eth/query/noevm.go index 6d252a7c9c..f7a7b37208 100644 --- a/eth/query/noevm.go +++ b/eth/query/noevm.go @@ -13,7 +13,7 @@ import ( ) func DeprecatedQueryChain( - _ string, _ store.BlockStore, _ appstate.ReadOnlyState, + _ string, _ store.BlockStore, _ state.ReadOnlyState, _ loomchain.ReadReceiptHandler, _ *evmaux.EvmAuxStore, _ uint64, ) ([]byte, error) { return nil, nil @@ -68,7 +68,7 @@ func GetTxByBlockAndIndex(_ store.BlockStore, _, _ uint64, _ *evmaux.EvmAuxStore } func QueryChain( - _ store.BlockStore, _ appstate.ReadOnlyState, _ eth.EthFilter, + _ store.BlockStore, _ state.ReadOnlyState, _ eth.EthFilter, _ loomchain.ReadReceiptHandler, _ *evmaux.EvmAuxStore, _ uint64, ) ([]*types.EthFilterLog, error) { return nil, nil diff --git a/evm/noevm.go b/evm/noevm.go index ec49e6c29b..d0ad707f4b 100644 --- a/evm/noevm.go +++ b/evm/noevm.go @@ -3,6 +3,8 @@ package evm import ( + "github.com/ethereum/go-ethereum/eth" + "github.com/loomnetwork/loomchain" "github.com/loomnetwork/loomchain/state" lvm "github.com/loomnetwork/loomchain/vm" @@ -17,7 +19,6 @@ const EVMEnabled = false func NewLoomVm( _ state.State, - _ loomchain.EventHandler, _ loomchain.WriteReceiptHandler, _ AccountBalanceManagerFactoryFunc, _ bool, diff --git a/rpc/eth/config.go b/rpc/eth/config.go deleted file mode 100644 index b55573c7e1..0000000000 --- a/rpc/eth/config.go +++ /dev/null @@ -1,14 +0,0 @@ -package eth - -// Web3Config contains settings that control the operation of the Web3 JSON-RPC method exposed -// via the /eth endpoint. -type Web3Config struct { - // GetLogsMaxBlockRange specifies the maximum number of blocks eth_getLogs will query per request - GetLogsMaxBlockRange uint64 -} - -func DefaultWeb3Config() *Web3Config { - return &Web3Config{ - GetLogsMaxBlockRange: 20, - } -} From 66aaf0813c88cceffd89c14f3deaf25ef5e073b7 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 10:27:09 +0000 Subject: [PATCH 40/49] makefile --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87472eda12..1ec42c90c2 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,9 @@ BINANCE_TGORACLE_DIR=$(GOPATH)/src/$(PKG_BINANCE_TGORACLE) # specific commit. GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. -#TG_GIT_REV = config -TG_GIT_REV = HEAD +TG_GIT_REV = config # loomnetwork/go-ethereum loomchain branch -#ETHEREUM_GIT_REV = ethapi -ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9 +ETHEREUM_GIT_REV = ethapi # use go-plugin we get 'timeout waiting for connection info' error HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961 LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9 From 48e37386b171506b48412dc5ba974d10f8528651 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 10:51:56 +0000 Subject: [PATCH 41/49] touch --- rpc/debug/trace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 1e143d9427..7c9025a62c 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -38,6 +38,7 @@ func TraceTransaction( if err != nil { return nil, errors.Wrapf(err, "getting block information at height %v", targetBlockNumber) } + tracer, err := createTracer(config) if err != nil { return nil, err From 4203aaf9b6dcfbcb61e9b55ab39987526ddc1e3d Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 12:14:09 +0000 Subject: [PATCH 42/49] touch --- rpc/debug/trace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 7c9025a62c..d0d5b1baaa 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -46,6 +46,7 @@ func TraceTransaction( if err := app.SetTracer(tracer, false); err != nil { return nil, err } + result := app.DeliverTx(block.Block.Data.Txs[txIndex]) switch tracer := tracer.(type) { From fff16343836d12ff503bf1ec3e409b00ab8632de Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 12:32:43 +0000 Subject: [PATCH 43/49] touch --- rpc/debug/trace.go | 1 - 1 file changed, 1 deletion(-) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index d0d5b1baaa..4f65a203e7 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -38,7 +38,6 @@ func TraceTransaction( if err != nil { return nil, errors.Wrapf(err, "getting block information at height %v", targetBlockNumber) } - tracer, err := createTracer(config) if err != nil { return nil, err From 3b8d97ea378724c7500ca2b34a9103fc3e82656f Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 13:19:24 +0000 Subject: [PATCH 44/49] touch --- rpc/debug/trace.go | 1 - 1 file changed, 1 deletion(-) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 4f65a203e7..1e143d9427 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -45,7 +45,6 @@ func TraceTransaction( if err := app.SetTracer(tracer, false); err != nil { return nil, err } - result := app.DeliverTx(block.Block.Data.Txs[txIndex]) switch tracer := tracer.(type) { From 9d2b80897c4987713b2b7f13ff31f0855f5c316e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 14:42:55 +0000 Subject: [PATCH 45/49] touch --- rpc/debug/trace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 1e143d9427..4f65a203e7 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -45,6 +45,7 @@ func TraceTransaction( if err := app.SetTracer(tracer, false); err != nil { return nil, err } + result := app.DeliverTx(block.Block.Data.Txs[txIndex]) switch tracer := tracer.(type) { From 6d95f1364cab9c9d662de65ce7a48b72c2199a53 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 15:24:02 +0000 Subject: [PATCH 46/49] lint fix --- auth/auth_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/auth/auth_test.go b/auth/auth_test.go index 839ca81c3e..ef657ccb2a 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -20,10 +20,6 @@ import ( "github.com/loomnetwork/loomchain/txhandler" ) -var nonceTxHandler = NonceHandler{nonceCache: make(map[string]uint64), lastHeight: 0} - -var nonceTxPostNonceMiddleware = txhandler.PostCommitMiddlewareFunc(nonceTxHandler.IncNonce) - func TestSignatureTxMiddleware(t *testing.T) { origBytes := []byte("hello") _, privKey, err := ed25519.GenerateKey(nil) From 6cf49dd792f08a4a007478e501f7d1222c03411e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 4 Nov 2019 15:59:53 +0000 Subject: [PATCH 47/49] lint fixes --- auth/multi_chain_sigtx_middleware_test.go | 1 - cmd/loom/gateway_reactors.go | 7 +++---- cmd/loom/loom.go | 18 ------------------ eth/polls/polls_test.go | 5 ++--- plugin/external.go | 2 +- plugin/loader.go | 2 +- receipts/leveldb/leveldb_test.go | 17 ++--------------- registry/v1/registry.go | 2 +- registry/v2/registry.go | 2 +- store/multi_writer_app_store.go | 4 ---- store/pruning_iavlstore.go | 8 -------- throttle/throttle.go | 6 ------ txhandler/middleware/deployer-middleware.go | 4 +--- 13 files changed, 12 insertions(+), 66 deletions(-) diff --git a/auth/multi_chain_sigtx_middleware_test.go b/auth/multi_chain_sigtx_middleware_test.go index e9bacdc98b..6dfdeb6ddc 100644 --- a/auth/multi_chain_sigtx_middleware_test.go +++ b/auth/multi_chain_sigtx_middleware_test.go @@ -35,7 +35,6 @@ import ( ) const ( - callId = uint32(2) seq = uint64(4) // todo defaultLoomChainId = "default" ) diff --git a/cmd/loom/gateway_reactors.go b/cmd/loom/gateway_reactors.go index e5be654f67..813240b170 100644 --- a/cmd/loom/gateway_reactors.go +++ b/cmd/loom/gateway_reactors.go @@ -17,10 +17,9 @@ import ( ) const ( - GatewayName = "gateway" - LoomGatewayName = "loomcoin-gateway" - BinanceGatewayName = "binance-gateway" - TronGatewayName = "tron-gateway" + GatewayName = "gateway" + LoomGatewayName = "loomcoin-gateway" + TronGatewayName = "tron-gateway" ) func startGatewayReactors( diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index b6cd0fc187..fcb55f7b3c 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -490,14 +490,6 @@ func contractInfoCommand() *cobra.Command { return cmd } -//nolint:deadcode -func recovery() { - if r := recover(); r != nil { - log.Error("caught RPC proxy exception, exiting", r) - os.Exit(1) - } -} - func startFeatureAutoEnabler( chainID string, cfg *config.ChainConfigConfig, nodeSigner glAuth.Signer, node backend.Backend, logger *loom.Logger, @@ -1051,16 +1043,6 @@ func getContractCtx(pluginName string, vmManager vm.Manager) func(state appstate } } -func getContractStaticCtx(pluginName string, vmManager vm.Manager) func(state appstate.State) (contractpb.StaticContext, error) { - return func(state appstate.State) (contractpb.StaticContext, error) { - pvm, err := vmManager.InitVM(vm.VMType_PLUGIN, state) - if err != nil { - return nil, err - } - return plugin.NewInternalContractContext(pluginName, pvm.(*plugin.PluginVM), true) - } -} - func initBackend(cfg *config.Config, abciServerAddr string, fnRegistry fnConsensus.FnRegistry) backend.Backend { ovCfg := &backend.OverrideConfig{ LogLevel: cfg.BlockchainLogLevel, diff --git a/eth/polls/polls_test.go b/eth/polls/polls_test.go index 75e4a31ba6..14f37b8c16 100644 --- a/eth/polls/polls_test.go +++ b/eth/polls/polls_test.go @@ -30,9 +30,8 @@ var ( ) const ( - deployId = uint32(1) - callId = uint32(2) - migrationTx = uint32(3) + deployId = uint32(1) + callId = uint32(2) ) func TestLogPoll(t *testing.T) { diff --git a/plugin/external.go b/plugin/external.go index 69e8c150b7..7d401fc57d 100644 --- a/plugin/external.go +++ b/plugin/external.go @@ -30,7 +30,7 @@ func (f *FileNameInfo) String() string { return f.Base + f.Ext + f.Version } -var fileInfoRE = regexp.MustCompile("(.+?)(\\.[a-zA-Z]+?)?\\.([0-9\\.]+)") +var fileInfoRE = regexp.MustCompile(`(.+?)(\.[a-zA-Z]+?)?\.([0-9\.]+)`) func parseFileName(name string) (*FileNameInfo, error) { groups := fileInfoRE.FindSubmatch([]byte(name)) diff --git a/plugin/loader.go b/plugin/loader.go index 4e2468925d..cbe960796c 100644 --- a/plugin/loader.go +++ b/plugin/loader.go @@ -121,7 +121,7 @@ func (m *StaticLoader) LoadContract(name string, blockHeight int64) (plugin.Cont } func (m *StaticLoader) findOverride(name string, blockHeight int64) plugin.Contract { - if overrides, _ := m.overrides[name]; overrides != nil { + if overrides := m.overrides[name]; overrides != nil { for i := len(overrides) - 1; i >= 0; i-- { if blockHeight >= overrides[i].BlockHeight { return overrides[i].Contract diff --git a/receipts/leveldb/leveldb_test.go b/receipts/leveldb/leveldb_test.go index e2fc61b10e..925691de3a 100644 --- a/receipts/leveldb/leveldb_test.go +++ b/receipts/leveldb/leveldb_test.go @@ -2,15 +2,15 @@ package leveldb import ( "bytes" - "fmt" "os" "testing" "github.com/gogo/protobuf/proto" "github.com/loomnetwork/go-loom/plugin/types" + "github.com/stretchr/testify/require" + "github.com/loomnetwork/loomchain/receipts/common" evmaux "github.com/loomnetwork/loomchain/store/evm_aux" - "github.com/stretchr/testify/require" ) const ( @@ -173,19 +173,6 @@ func TestConfirmTransactionReceipts(t *testing.T) { require.Error(t, err) } -//nolint:deadcode -func dumpDbEntries(evmAuxStore *evmaux.EvmAuxStore) error { - fmt.Println("\nDumping leveldb") - db := evmAuxStore.DB() - iter := db.NewIterator(nil, nil) - defer iter.Release() - for iter.Next() { - fmt.Printf("key %s\t\tvalue %s", iter.Key(), iter.Value()) - } - fmt.Println() - return iter.Error() -} - func countDbEntries(evmAuxStore *evmaux.EvmAuxStore) (uint64, error) { count := uint64(0) db := evmAuxStore.DB() diff --git a/registry/v1/registry.go b/registry/v1/registry.go index b65d2feb39..cd7dfcd568 100644 --- a/registry/v1/registry.go +++ b/registry/v1/registry.go @@ -18,7 +18,7 @@ const ( ) var ( - validNameRE = regexp.MustCompile("^[a-zA-Z0-9\\.\\-]+$") + validNameRE = regexp.MustCompile(`^[a-zA-Z0-9\.\-]+$`) ) func recordKey(name string) []byte { diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 70770aa13e..50ca02fed0 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -19,7 +19,7 @@ const ( ) var ( - validNameRE = regexp.MustCompile("^[a-zA-Z0-9\\.\\-]+$") + validNameRE = regexp.MustCompile(`^[a-zA-Z0-9\.\-]+$`) // Store Keys contractAddrKeyPrefix = []byte("reg_caddr") diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index a2fd6475ce..db133db74e 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -19,10 +19,6 @@ import ( "github.com/loomnetwork/loomchain/features" ) -const ( - evmCacheRoots = 100 -) - var ( // This is the same prefix as vmPrefix in evm/loomevm.go // We have to do this to avoid cyclic dependency diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 7f452f445d..cff86bfd89 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -266,11 +266,3 @@ func (s *PruningIAVLStore) loopWithInterval(step func() error, interval time.Dur time.Sleep(interval) } } - -type pruningIAVLStoreSnapshot struct { - *PruningIAVLStore -} - -func (s *pruningIAVLStoreSnapshot) Release() { - // noop -} diff --git a/throttle/throttle.go b/throttle/throttle.go index 39bbe25ae5..33f136069c 100644 --- a/throttle/throttle.go +++ b/throttle/throttle.go @@ -20,12 +20,6 @@ import ( appstate "github.com/loomnetwork/loomchain/state" ) -const ( - deployId = uint32(1) - callId = uint32(2) - migrationId = uint32(3) -) - type Throttle struct { MaxCallCount int64 sessionDuration int64 diff --git a/txhandler/middleware/deployer-middleware.go b/txhandler/middleware/deployer-middleware.go index 64b9c69c65..3099f4a9f1 100644 --- a/txhandler/middleware/deployer-middleware.go +++ b/txhandler/middleware/deployer-middleware.go @@ -20,9 +20,7 @@ import ( ) const ( - deployId = uint32(1) - callId = uint32(2) - migrationId = uint32(3) + callId = uint32(2) ) var ( From 4b609deef5e02ecf30371f944b9b064a959391e8 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 6 Nov 2019 10:32:11 +0000 Subject: [PATCH 48/49] touch --- rpc/debug/trace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/debug/trace.go b/rpc/debug/trace.go index 4f65a203e7..d0d5b1baaa 100644 --- a/rpc/debug/trace.go +++ b/rpc/debug/trace.go @@ -38,6 +38,7 @@ func TraceTransaction( if err != nil { return nil, errors.Wrapf(err, "getting block information at height %v", targetBlockNumber) } + tracer, err := createTracer(config) if err != nil { return nil, err From a1d93affe4a2a75bbe28cb4014b22c6e1470bb99 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Thu, 7 Nov 2019 09:49:53 +0000 Subject: [PATCH 49/49] range fix --- store/multi_writer_app_store.go | 57 +++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/store/multi_writer_app_store.go b/store/multi_writer_app_store.go index db133db74e..e28acefad9 100644 --- a/store/multi_writer_app_store.go +++ b/store/multi_writer_app_store.go @@ -17,6 +17,7 @@ import ( "github.com/loomnetwork/loomchain/db" "github.com/loomnetwork/loomchain/features" + "github.com/loomnetwork/loomchain/log" ) var ( @@ -322,24 +323,54 @@ func (s *multiWriterStoreSnapshot) Range(prefix []byte) plugin.RangeData { panic(errors.New("Range over nil prefix not implemented")) } - var data plugin.RangeData + ret := make(plugin.RangeData, 0) + if bytes.Equal(prefix, vmPrefix) || util.HasPrefix(prefix, vmPrefix) { - for iter := s.evmDbSnapshot.NewIterator(prefix, nil); iter.Valid(); iter.Next() { - if 0 != bytes.Compare(prefix, iter.Key()[:len(prefix)]) { - break + it := s.evmDbSnapshot.NewIterator(prefix, prefixRangeEnd(prefix)) + defer it.Close() + + for ; it.Valid(); it.Next() { + key := it.Key() + if util.HasPrefix(key, prefix) { + var err error + key, err = util.UnprefixKey(key, prefix) + if err != nil { + panic(err) + } + + ret = append(ret, &plugin.RangeEntry{ + Key: key, + Value: it.Value(), + }) } - data = append(data, &plugin.RangeEntry{iter.Key(), iter.Value()}) } - } else { - prefix = append(prefix, 0) - s.appStoreTree.IterateRangeInclusive(prefix, nil, true, func(key []byte, value []byte, _ int64) bool { - if len(key) < len(prefix) || 0 != bytes.Compare(prefix, key[:len(prefix)]) { - return true + return ret + } + + // Otherwise iterate over the IAVL tree + keys, values, _, err := s.appStoreTree.GetRangeWithProof(prefix, prefixRangeEnd(prefix), 0) + if err != nil { + log.Error("failed to get range", "prefix", string(prefix), "err", err) + return ret + } + + for i, k := range keys { + // Tree range gives all keys that has prefix but it does not check zero byte + // after the prefix. So we have to check zero byte after prefix using util.HasPrefix + if util.HasPrefix(k, prefix) { + k, err = util.UnprefixKey(k, prefix) + if err != nil { + panic(err) } - data = append(data, &plugin.RangeEntry{key, value}) - return false + } else { // Skip this key as it does not have the prefix + continue + } + + ret = append(ret, &plugin.RangeEntry{ + Key: k, + Value: values[i], }) } - return data + return ret }