From 305220e7db9caf899f48e897761b9873e74e1af5 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Tue, 10 Aug 2021 17:33:18 +0200 Subject: [PATCH 01/15] Add skeleton baseledger p2p client --- network/network.go | 2 + network/node.go | 2 + network/p2p/baseledger.go | 102 ++++++++++++++++++++++++++++++++++++++ network/p2p/common.go | 3 ++ 4 files changed, 109 insertions(+) create mode 100644 network/p2p/baseledger.go diff --git a/network/network.go b/network/network.go index 976d1876..c4cad164 100644 --- a/network/network.go +++ b/network/network.go @@ -897,6 +897,8 @@ func (n *Network) P2PAPIClient() (p2p.API, error) { apiClient = p2p.InitParityP2PProvider(common.StringOrNil(rpcURL), n.ID.String(), n) case p2p.ProviderQuorum: apiClient = p2p.InitQuorumP2PProvider(common.StringOrNil(rpcURL), n.ID.String(), n) + case p2p.ProviderBaseledger: + apiClient = p2p.InitBaseledgerP2PProvider(common.StringOrNil(rpcURL), n.ID.String(), n) default: return nil, fmt.Errorf("Failed to resolve p2p provider for network %s; unsupported client", n.ID) } diff --git a/network/node.go b/network/node.go index d872c72a..09054d76 100644 --- a/network/node.go +++ b/network/node.go @@ -749,6 +749,8 @@ func (n *Node) P2PAPIClient() (p2p.API, error) { apiClient = p2p.InitParityP2PProvider(rpcURL, n.NetworkID.String(), n.Network) case p2p.ProviderQuorum: apiClient = p2p.InitQuorumP2PProvider(rpcURL, n.NetworkID.String(), n.Network) + case p2p.ProviderBaseledger: + apiClient = p2p.InitBaseledgerP2PProvider(rpcURL, n.NetworkID.String(), n.Network) default: return nil, fmt.Errorf("Failed to resolve p2p provider for network node %s; unsupported client", n.ID) } diff --git a/network/p2p/baseledger.go b/network/p2p/baseledger.go new file mode 100644 index 00000000..63334e11 --- /dev/null +++ b/network/p2p/baseledger.go @@ -0,0 +1,102 @@ +package p2p + +import ( + "errors" + "math/big" + "strings" + + "github.com/jinzhu/gorm" + uuid "github.com/kthomas/go.uuid" + "github.com/provideplatform/nchain/common" + provide "github.com/provideplatform/provide-go/api/nchain" +) + +// BaseledgerP2PProvider is a network.p2p.API implementing the baseledger API +type BaseledgerP2PProvider struct { + rpcClientKey *string + rpcURL *string + network common.Configurable + networkID string +} + +// BaseledgerP2PProvider initializes and returns the baseledger p2p provider +func InitBaseledgerP2PProvider(rpcURL *string, networkID string, ntwrk common.Configurable) *BaseledgerP2PProvider { + return &BaseledgerP2PProvider{ + rpcClientKey: rpcURL, + rpcURL: rpcURL, + network: ntwrk, + networkID: networkID, + } +} + +// DefaultEntrypoint returns the default entrypoint to run when starting the container, when one is not otherwise provided +func (p *BaseledgerP2PProvider) DefaultEntrypoint() []string { + return []string{} +} + +// EnrichStartCommand returns the cmd to append to the command to start the container +func (p *BaseledgerP2PProvider) EnrichStartCommand(bootnodes []string) []string { + cmd := make([]string, 0) + + return cmd +} + +// AcceptNonReservedPeers allows non-reserved peers to connect +func (p *BaseledgerP2PProvider) AcceptNonReservedPeers() error { + return errors.New("not yet implemented") +} + +// DropNonReservedPeers only allows reserved peers to connect; reversed by calling `AcceptNonReservedPeers` +func (p *BaseledgerP2PProvider) DropNonReservedPeers() error { + return errors.New("not yet implemented") +} + +// AddPeer adds a peer by its peer url +func (p *BaseledgerP2PProvider) AddPeer(peerURL string) error { + return errors.New("not yet implemented") +} + +// FetchTxReceipt fetch a transaction receipt given its hash +func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { + return nil, errors.New("not yet implemented") +} + +// FetchTxTraces fetch transaction traces given its hash +func (p *BaseledgerP2PProvider) FetchTxTraces(hash string) (*provide.TxTrace, error) { + return nil, errors.New("not yet implemented") +} + +// FormatBootnodes formats the given peer urls as a valid bootnodes param +func (p *BaseledgerP2PProvider) FormatBootnodes(bootnodes []string) string { + return strings.Join(bootnodes, ",") +} + +// ParsePeerURL parses a peer url from the given raw log string +func (p *BaseledgerP2PProvider) ParsePeerURL(string) (*string, error) { + return nil, errors.New("not yet implemented") +} + +// RemovePeer removes a peer by its peer url +func (p *BaseledgerP2PProvider) RemovePeer(peerURL string) error { + return errors.New("not yet implemented") +} + +// ResolvePeerURL attempts to resolve one or more viable peer urls +func (p *BaseledgerP2PProvider) ResolvePeerURL() (*string, error) { + return nil, errors.New("not yet implemented") +} + +// ResolveTokenContract attempts to resolve the given token contract details for the contract at a given address +func (p *BaseledgerP2PProvider) ResolveTokenContract(signerAddress string, receipt interface{}, artifact *provide.CompiledArtifact) (*string, *string, *big.Int, *string, error) { + return nil, nil, nil, nil, errors.New("not yet implemented") +} + +// RequireBootnodes attempts to resolve the peers to use as bootnodes +func (p *BaseledgerP2PProvider) RequireBootnodes(db *gorm.DB, userID *uuid.UUID, networkID *uuid.UUID, n common.Configurable) error { + return errors.New("not yet implemented") +} + +// Upgrade executes a pending upgrade +func (p *BaseledgerP2PProvider) Upgrade() error { + return errors.New("not yet implemented") +} diff --git a/network/p2p/common.go b/network/p2p/common.go index d8f085e3..01bfbf1d 100644 --- a/network/p2p/common.go +++ b/network/p2p/common.go @@ -58,6 +58,9 @@ const ProviderParity = "parity" // ProviderQuorum quorum p2p provider const ProviderQuorum = "quorum" +// ProviderBaseledger baseledger p2p provider +const ProviderBaseledger = "baseledger" + const tokenTypeERC20 = "ERC-20" const tokenTypeERC721 = "ERC-721" From 962a62ca0329ddeaf0acf0e673a050aa6a06115d Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Wed, 11 Aug 2021 18:31:04 +0200 Subject: [PATCH 02/15] Add first version of tx receipt --- network/handlers.go | 26 +++++++ network/p2p/baseledger.go | 160 ++++++++++++++++++++++++++++++++++++-- network/p2p/fabric.go | 2 +- 3 files changed, 182 insertions(+), 6 deletions(-) diff --git a/network/handlers.go b/network/handlers.go index 7553bc9a..41b81db1 100644 --- a/network/handlers.go +++ b/network/handlers.go @@ -38,6 +38,32 @@ func InstallNetworksAPI(r *gin.Engine) { r.DELETE("/api/v1/networks/:id/nodes/:nodeId", deleteNodeHandler) r.GET("/api/v1/networks/:id/oracles", networkOraclesListHandler) + r.GET("/api/v1/tendermintrpc/:id/:txHash", testTendermintTxReceiptHandler) +} + +// TODO: just used as entry point for testing p2p client +func testTendermintTxReceiptHandler(c *gin.Context) { + network := &Network{} + dbconf.DatabaseConnection().Where("id = ?", c.Param("id")).Find(&network) + if network.ID == uuid.Nil { + provide.RenderError("network not found", 404, c) + return + } + + p2pClient, err := network.P2PAPIClient() + if err != nil { + provide.RenderError(err.Error(), 500, c) + return + } + + txReceipt, err := p2pClient.FetchTxReceipt("", c.Param("txHash")) + + if err != nil { + provide.RenderError(err.Error(), 500, c) + return + } + + provide.Render(txReceipt, 200, c) } func createNetworkHandler(c *gin.Context) { diff --git a/network/p2p/baseledger.go b/network/p2p/baseledger.go index 63334e11..9d991928 100644 --- a/network/p2p/baseledger.go +++ b/network/p2p/baseledger.go @@ -1,14 +1,19 @@ package p2p import ( + "encoding/json" "errors" "math/big" + "strconv" "strings" + "time" + ethCommon "github.com/ethereum/go-ethereum/common" "github.com/jinzhu/gorm" uuid "github.com/kthomas/go.uuid" "github.com/provideplatform/nchain/common" - provide "github.com/provideplatform/provide-go/api/nchain" + provide "github.com/provideplatform/provide-go/api" + nchain "github.com/provideplatform/provide-go/api/nchain" ) // BaseledgerP2PProvider is a network.p2p.API implementing the baseledger API @@ -57,12 +62,65 @@ func (p *BaseledgerP2PProvider) AddPeer(peerURL string) error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { - return nil, errors.New("not yet implemented") +func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*nchain.TxReceipt, error) { + httpClient := &provide.Client{ + Host: *p.rpcURL, + Scheme: "http", + } + + status, resp, err := httpClient.Get("tx", map[string]interface{}{"hash": hash}) + + if err != nil { + return nil, err + } + + if status != 200 { + respJSON, _ := json.Marshal(resp) + return nil, errors.New(string(respJSON)) + } + + txEntity := &TendermintTx{} + respJSON, _ := json.Marshal(resp) + json.Unmarshal(respJSON, &txEntity) + + status, resp, err = httpClient.Get("block", map[string]interface{}{"height": txEntity.Result.Height}) + + if err != nil { + return nil, err + } + + if status != 200 { + respJSON, _ := json.Marshal(resp) + return nil, errors.New(string(respJSON)) + } + + blockEntity := &TendermintBlock{} + respJSON, _ = json.Marshal(resp) + json.Unmarshal(respJSON, &blockEntity) + + gasUsed, _ := strconv.Atoi(txEntity.Result.TxResult.GasUsed) + n := new(big.Int) + blockNumber, _ := n.SetString(txEntity.Result.Height, 10) + var logs []interface{} + json.Unmarshal([]byte(txEntity.Result.TxResult.Log), &logs) + return &nchain.TxReceipt{ + // TODO: tx receipt is heavy coupled with ethereum, fix this? + TxHash: ethCommon.HexToHash(txEntity.Result.Hash), + ContractAddress: ethCommon.Address{}, + GasUsed: uint64(gasUsed), + BlockHash: ethCommon.HexToHash(blockEntity.Result.BlockID.Hash), + BlockNumber: blockNumber, + TransactionIndex: 0, + PostState: nil, + Status: uint64(txEntity.Result.TxResult.Code), + CumulativeGasUsed: uint64(gasUsed), + Bloom: nil, + Logs: logs, + }, nil } // FetchTxTraces fetch transaction traces given its hash -func (p *BaseledgerP2PProvider) FetchTxTraces(hash string) (*provide.TxTrace, error) { +func (p *BaseledgerP2PProvider) FetchTxTraces(hash string) (*nchain.TxTrace, error) { return nil, errors.New("not yet implemented") } @@ -87,7 +145,7 @@ func (p *BaseledgerP2PProvider) ResolvePeerURL() (*string, error) { } // ResolveTokenContract attempts to resolve the given token contract details for the contract at a given address -func (p *BaseledgerP2PProvider) ResolveTokenContract(signerAddress string, receipt interface{}, artifact *provide.CompiledArtifact) (*string, *string, *big.Int, *string, error) { +func (p *BaseledgerP2PProvider) ResolveTokenContract(signerAddress string, receipt interface{}, artifact *nchain.CompiledArtifact) (*string, *string, *big.Int, *string, error) { return nil, nil, nil, nil, errors.New("not yet implemented") } @@ -100,3 +158,95 @@ func (p *BaseledgerP2PProvider) RequireBootnodes(db *gorm.DB, userID *uuid.UUID, func (p *BaseledgerP2PProvider) Upgrade() error { return errors.New("not yet implemented") } + +// TODO: should move to provide-go models? +type TendermintTx struct { + Jsonrpc string `json:"jsonrpc"` + ID int `json:"id"` + Result struct { + Hash string `json:"hash"` + Height string `json:"height"` + Index int `json:"index"` + TxResult struct { + Code int `json:"code"` + Data string `json:"data"` + Log string `json:"log"` + Info string `json:"info"` + GasWanted string `json:"gas_wanted"` + GasUsed string `json:"gas_used"` + Events []struct { + Type string `json:"type"` + Attributes []struct { + Key string `json:"key"` + Value string `json:"value"` + Index bool `json:"index"` + } `json:"attributes"` + } `json:"events"` + Codespace string `json:"codespace"` + } `json:"tx_result"` + Tx string `json:"tx"` + } `json:"result"` +} + +type TendermintBlock struct { + Jsonrpc string `json:"jsonrpc"` + ID int `json:"id"` + Result struct { + BlockID struct { + Hash string `json:"hash"` + Parts struct { + Total int `json:"total"` + Hash string `json:"hash"` + } `json:"parts"` + } `json:"block_id"` + Block struct { + Header struct { + Version struct { + Block string `json:"block"` + } `json:"version"` + ChainID string `json:"chain_id"` + Height string `json:"height"` + Time time.Time `json:"time"` + LastBlockID struct { + Hash string `json:"hash"` + Parts struct { + Total int `json:"total"` + Hash string `json:"hash"` + } `json:"parts"` + } `json:"last_block_id"` + LastCommitHash string `json:"last_commit_hash"` + DataHash string `json:"data_hash"` + ValidatorsHash string `json:"validators_hash"` + NextValidatorsHash string `json:"next_validators_hash"` + ConsensusHash string `json:"consensus_hash"` + AppHash string `json:"app_hash"` + LastResultsHash string `json:"last_results_hash"` + EvidenceHash string `json:"evidence_hash"` + ProposerAddress string `json:"proposer_address"` + } `json:"header"` + Data struct { + Txs []string `json:"txs"` + } `json:"data"` + Evidence struct { + Evidence []interface{} `json:"evidence"` + } `json:"evidence"` + LastCommit struct { + Height string `json:"height"` + Round int `json:"round"` + BlockID struct { + Hash string `json:"hash"` + Parts struct { + Total int `json:"total"` + Hash string `json:"hash"` + } `json:"parts"` + } `json:"block_id"` + Signatures []struct { + BlockIDFlag int `json:"block_id_flag"` + ValidatorAddress string `json:"validator_address"` + Timestamp time.Time `json:"timestamp"` + Signature string `json:"signature"` + } `json:"signatures"` + } `json:"last_commit"` + } `json:"block"` + } `json:"result"` +} diff --git a/network/p2p/fabric.go b/network/p2p/fabric.go index 6735b293..0fa633d7 100644 --- a/network/p2p/fabric.go +++ b/network/p2p/fabric.go @@ -49,7 +49,7 @@ func (p *HyperledgerFabricP2PProvider) EnrichStartCommand(bootnodes []string) [] } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(hash, signerAddress string) (*provide.TxReceipt, error) { +func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { return nil, errors.New("fabric does not impl FetchTxReceipt()") } From d207ec4764b54c1f0d812cc326c8b23cd6e2e7d0 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Wed, 18 Aug 2021 03:00:43 +0200 Subject: [PATCH 03/15] Add baseledger network stats data source factory --- cmd/statsdaemon/stats_daemon.go | 168 ++++++++++++++++++++++++++++++++ network/network.go | 11 +++ 2 files changed, 179 insertions(+) diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index e15b39f3..33938fbc 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -333,6 +333,172 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta } } +// TMP struct, just for testing +// format is still unknown +type BaseledgerSubscriptionResponse struct { + ID interface{} `json:"id"` + Jsonrpc string `json:"jsonrpc"` + Result map[string]interface{} `json:"result"` +} + +// TMP struct, just for testing +// format is still unknown +type BaseledgerBlockHeader struct { + AppHash string `json:"app_hash"` + ChainID string `json:"chain_id"` + ConsensusHash string `json:"consensus_hash"` + DataHash string `json:"data_hash"` + EvidenceHash string `json:"evidence_hash"` + Height string `json:"height"` + LastBlockID struct { + Hash string `json:"hash"` + Parts struct { + Hash string `json:"hash"` + Total int `json:"total"` + } `json:"parts"` + } `json:"last_block_id"` + LastCommitHash string `json:"last_commit_hash"` + LastResultsHash string `json:"last_results_hash"` + NextValidatorsHash string `json:"next_validators_hash"` + ProposerAddress string `json:"proposer_address"` + Time time.Time `json:"time"` + ValidatorsHash string `json:"validators_hash"` + Version struct { + App string `json:"app"` + Block string `json:"block"` + } `json:"version"` +} + +// TMP struct, just for testing +// format is still unknown +type BaseledgerBlockHeaderResponse struct { + ID int `json:"id"` + Jsonrpc string `json:"jsonrpc"` + Result struct { + Data struct { + Type string `json:"type"` + Value struct { + Header struct { + AppHash string `json:"app_hash"` + ChainID string `json:"chain_id"` + ConsensusHash string `json:"consensus_hash"` + DataHash string `json:"data_hash"` + EvidenceHash string `json:"evidence_hash"` + Height string `json:"height"` + LastBlockID struct { + Hash string `json:"hash"` + Parts struct { + Hash string `json:"hash"` + Total int `json:"total"` + } `json:"parts"` + } `json:"last_block_id"` + LastCommitHash string `json:"last_commit_hash"` + LastResultsHash string `json:"last_results_hash"` + NextValidatorsHash string `json:"next_validators_hash"` + ProposerAddress string `json:"proposer_address"` + Time time.Time `json:"time"` + ValidatorsHash string `json:"validators_hash"` + Version struct { + App string `json:"app"` + Block string `json:"block"` + } `json:"version"` + } `json:"header"` + NumTxs string `json:"num_txs"` + ResultBeginBlock struct { + Events []struct { + Attributes []struct { + Index bool `json:"index"` + Key string `json:"key"` + Value string `json:"value"` + } `json:"attributes"` + Type string `json:"type"` + } `json:"events"` + } `json:"result_begin_block"` + ResultEndBlock struct { + ValidatorUpdates interface{} `json:"validator_updates"` + } `json:"result_end_block"` + } `json:"value"` + } `json:"data"` + Events struct { + BlockHeader []time.Time `json:"block.header"` + TmEvent []string `json:"tm.event"` + } `json:"events"` + Query string `json:"query"` + } `json:"result"` +} + +// BaseledgerNetworkStatsDataSourceFactory builds and returns a JSON-RPC and streaming websocket +// data source which is used by stats daemon instances to consume EVM-based network statistics +func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsDataSource { + return &NetworkStatsDataSource{ + Network: network, + + Poll: func(ch chan *provide.NetworkStatus) error { + return new(jsonRpcNotSupported) + }, + + Stream: func(ch chan *provide.NetworkStatus) error { + websocketURL := network.WebsocketURL() + if websocketURL == "" { + err := new(websocketNotSupported) + return *err + } + var wsDialer websocket.Dialer + wsConn, _, err := wsDialer.Dial(websocketURL, nil) + if err != nil { + common.Log.Errorf("Failed to establish network stats websocket connection to %s; %s", websocketURL, err.Error()) + } else { + defer wsConn.Close() + // { "jsonrpc": "2.0", "method": "subscribe", "params": ["tm.event='NewBlock'"], "id": 1 } + payload := map[string]interface{}{ + "method": "subscribe", + "params": []string{"tm.event='NewBlockHeader'"}, + "id": 1, + "jsonrpc": "2.0", + } + if err := wsConn.WriteJSON(payload); err != nil { + common.Log.Errorf("Failed to write subscribe message to network stats websocket connection") + } else { + common.Log.Debugf("Subscribed to network stats websocket: %s", websocketURL) + for { + _, message, err := wsConn.ReadMessage() + if err != nil { + common.Log.Errorf("Failed to receive message on network stats websocket; %s", err) + break + } else { + common.Log.Debugf("Received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) + // TODO: try unmarshaling whole struct, for some reason parsing time is failing in header + // even though tendermint timestamps layout is RFC3339 + response := &BaseledgerSubscriptionResponse{} + err := json.Unmarshal(message, response) + if err != nil { + common.Log.Warningf("Failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) + } else { + if result, ok := response.Result["data"].(map[string]interface{}); ok { + if resultJSON, err := json.Marshal(result); err == nil { + header := &BaseledgerBlockHeader{} + err := json.Unmarshal(resultJSON, header) + if err != nil { + common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) + } else if header != nil { + ch <- &provide.NetworkStatus{ + Meta: map[string]interface{}{ + "last_block_header": result, + }, + } + } + } + } + } + } + } + } + } + return err + }, + } +} + // Consume the websocket stream; attempts to fallback to JSON-RPC if websocket stream fails or is not available for the network func (sd *StatsDaemon) consume() []error { errs := make([]error, 0) @@ -700,6 +866,8 @@ func NewNetworkStatsDaemon(lg *logger.Logger, network *network.Network) *StatsDa sd.dataSource = BcoinNetworkStatsDataSourceFactory(network) } else if network.IsEthereumNetwork() { sd.dataSource = EthereumNetworkStatsDataSourceFactory(network) + } else if network.IsBaseledgerNetwork() { + sd.dataSource = BaseledgerNetworkStatsDataSourceFactory(network) } //sd.handleSignals() diff --git a/network/network.go b/network/network.go index c4cad164..923c6b99 100644 --- a/network/network.go +++ b/network/network.go @@ -55,6 +55,7 @@ const networkConfigIsHandshakeNetwork = "is_handshake_network" const networkConfigIsHyperledgerBesuNetwork = "is_hyperledger_besu_network" const networkConfigIsHyperledgerFabricNetwork = "is_hyperledger_fabric_network" const networkConfigIsQuorumNetwork = "is_quorum_network" +const networkConfigIsBaseledgerNetwork = "is_baseledger_network" const networkConfigEnvBootnodes = "BOOTNODES" const networkConfigEnvClient = "CLIENT" @@ -868,6 +869,16 @@ func (n *Network) IsHandshakeNetwork() bool { return false } +func (n *Network) IsBaseledgerNetwork() bool { + cfg := n.ParseConfig() + if cfg != nil { + if isBaseledgerNetwork, ok := cfg[networkConfigIsBaseledgerNetwork].(bool); ok { + return isBaseledgerNetwork + } + } + return false +} + // P2PAPIClient returns an instance of the network's underlying p2p.API, if that is possible given the network config func (n *Network) P2PAPIClient() (p2p.API, error) { cfg := n.ParseConfig() From be359c8f9a505f589b14eb57fbda3a34ff1baac5 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Fri, 20 Aug 2021 15:58:28 +0200 Subject: [PATCH 04/15] Make tx receipt more generic --- cmd/statsdaemon/stats_daemon.go | 58 --------------------------------- network/handlers.go | 26 --------------- network/p2p/baseledger.go | 12 +++---- network/p2p/common.go | 17 +++++++++- network/p2p/fabric.go | 2 +- network/p2p/geth.go | 10 +++--- network/p2p/nethermind.go | 10 +++--- network/p2p/parity.go | 10 +++--- network/p2p/quorum.go | 10 +++--- tx/shuttle_consumer.go | 8 ++++- tx/tx.go | 28 +++++++++++----- 11 files changed, 68 insertions(+), 123 deletions(-) diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index 33938fbc..8d49aee4 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -369,64 +369,6 @@ type BaseledgerBlockHeader struct { } `json:"version"` } -// TMP struct, just for testing -// format is still unknown -type BaseledgerBlockHeaderResponse struct { - ID int `json:"id"` - Jsonrpc string `json:"jsonrpc"` - Result struct { - Data struct { - Type string `json:"type"` - Value struct { - Header struct { - AppHash string `json:"app_hash"` - ChainID string `json:"chain_id"` - ConsensusHash string `json:"consensus_hash"` - DataHash string `json:"data_hash"` - EvidenceHash string `json:"evidence_hash"` - Height string `json:"height"` - LastBlockID struct { - Hash string `json:"hash"` - Parts struct { - Hash string `json:"hash"` - Total int `json:"total"` - } `json:"parts"` - } `json:"last_block_id"` - LastCommitHash string `json:"last_commit_hash"` - LastResultsHash string `json:"last_results_hash"` - NextValidatorsHash string `json:"next_validators_hash"` - ProposerAddress string `json:"proposer_address"` - Time time.Time `json:"time"` - ValidatorsHash string `json:"validators_hash"` - Version struct { - App string `json:"app"` - Block string `json:"block"` - } `json:"version"` - } `json:"header"` - NumTxs string `json:"num_txs"` - ResultBeginBlock struct { - Events []struct { - Attributes []struct { - Index bool `json:"index"` - Key string `json:"key"` - Value string `json:"value"` - } `json:"attributes"` - Type string `json:"type"` - } `json:"events"` - } `json:"result_begin_block"` - ResultEndBlock struct { - ValidatorUpdates interface{} `json:"validator_updates"` - } `json:"result_end_block"` - } `json:"value"` - } `json:"data"` - Events struct { - BlockHeader []time.Time `json:"block.header"` - TmEvent []string `json:"tm.event"` - } `json:"events"` - Query string `json:"query"` - } `json:"result"` -} - // BaseledgerNetworkStatsDataSourceFactory builds and returns a JSON-RPC and streaming websocket // data source which is used by stats daemon instances to consume EVM-based network statistics func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsDataSource { diff --git a/network/handlers.go b/network/handlers.go index 41b81db1..7553bc9a 100644 --- a/network/handlers.go +++ b/network/handlers.go @@ -38,32 +38,6 @@ func InstallNetworksAPI(r *gin.Engine) { r.DELETE("/api/v1/networks/:id/nodes/:nodeId", deleteNodeHandler) r.GET("/api/v1/networks/:id/oracles", networkOraclesListHandler) - r.GET("/api/v1/tendermintrpc/:id/:txHash", testTendermintTxReceiptHandler) -} - -// TODO: just used as entry point for testing p2p client -func testTendermintTxReceiptHandler(c *gin.Context) { - network := &Network{} - dbconf.DatabaseConnection().Where("id = ?", c.Param("id")).Find(&network) - if network.ID == uuid.Nil { - provide.RenderError("network not found", 404, c) - return - } - - p2pClient, err := network.P2PAPIClient() - if err != nil { - provide.RenderError(err.Error(), 500, c) - return - } - - txReceipt, err := p2pClient.FetchTxReceipt("", c.Param("txHash")) - - if err != nil { - provide.RenderError(err.Error(), 500, c) - return - } - - provide.Render(txReceipt, 200, c) } func createNetworkHandler(c *gin.Context) { diff --git a/network/p2p/baseledger.go b/network/p2p/baseledger.go index 9d991928..348f7e27 100644 --- a/network/p2p/baseledger.go +++ b/network/p2p/baseledger.go @@ -8,7 +8,6 @@ import ( "strings" "time" - ethCommon "github.com/ethereum/go-ethereum/common" "github.com/jinzhu/gorm" uuid "github.com/kthomas/go.uuid" "github.com/provideplatform/nchain/common" @@ -62,7 +61,7 @@ func (p *BaseledgerP2PProvider) AddPeer(peerURL string) error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*nchain.TxReceipt, error) { +func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { httpClient := &provide.Client{ Host: *p.rpcURL, Scheme: "http", @@ -103,12 +102,11 @@ func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*nch blockNumber, _ := n.SetString(txEntity.Result.Height, 10) var logs []interface{} json.Unmarshal([]byte(txEntity.Result.TxResult.Log), &logs) - return &nchain.TxReceipt{ - // TODO: tx receipt is heavy coupled with ethereum, fix this? - TxHash: ethCommon.HexToHash(txEntity.Result.Hash), - ContractAddress: ethCommon.Address{}, + return &TxReceipt{ + TxHash: []byte(txEntity.Result.Hash), + ContractAddress: nil, GasUsed: uint64(gasUsed), - BlockHash: ethCommon.HexToHash(blockEntity.Result.BlockID.Hash), + BlockHash: []byte(blockEntity.Result.BlockID.Hash), BlockNumber: blockNumber, TransactionIndex: 0, PostState: nil, diff --git a/network/p2p/common.go b/network/p2p/common.go index 01bfbf1d..f72520d7 100644 --- a/network/p2p/common.go +++ b/network/p2p/common.go @@ -71,7 +71,7 @@ type API interface { AddPeer(string) error RemovePeer(string) error ParsePeerURL(string) (*string, error) - FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) + FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) FetchTxTraces(hash string) (*provide.TxTrace, error) FormatBootnodes([]string) string RequireBootnodes(db *gorm.DB, userID *uuid.UUID, networkID *uuid.UUID, n common.Configurable) error @@ -83,6 +83,21 @@ type API interface { EnrichStartCommand(bootnodes []string) []string } +// move to provide-go +type TxReceipt struct { + TxHash []byte `json:"hash"` + ContractAddress []byte `json:"contract_address"` + GasUsed uint64 `json:"gas_used"` + BlockHash []byte `json:"block_hash,omitempty"` + BlockNumber *big.Int `json:"block,omitempty"` + TransactionIndex uint `json:"transaction_index"` + PostState []byte `json:"root"` + Status uint64 `json:"status"` + CumulativeGasUsed uint64 `json:"cumulative_gas_used"` + Bloom interface{} `json:"logs_bloom"` + Logs []interface{} `json:"logs"` +} + func evmFetchTxReceipt(rpcClientKey, rpcURL, signerAddress, hash string) (*types.Receipt, error) { receipt, err := providecrypto.EVMGetTxReceipt(rpcClientKey, rpcURL, hash, signerAddress) if err != nil { diff --git a/network/p2p/fabric.go b/network/p2p/fabric.go index 0fa633d7..23ac8acf 100644 --- a/network/p2p/fabric.go +++ b/network/p2p/fabric.go @@ -49,7 +49,7 @@ func (p *HyperledgerFabricP2PProvider) EnrichStartCommand(bootnodes []string) [] } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { +func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { return nil, errors.New("fabric does not impl FetchTxReceipt()") } diff --git a/network/p2p/geth.go b/network/p2p/geth.go index 92fd59b0..6ba469de 100644 --- a/network/p2p/geth.go +++ b/network/p2p/geth.go @@ -95,7 +95,7 @@ func (p *GethP2PProvider) EnrichStartCommand(bootnodes []string) []string { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { +func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -106,11 +106,11 @@ func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.T logs = append(logs, *log) } - return &provide.TxReceipt{ - TxHash: receipt.TxHash, - ContractAddress: receipt.ContractAddress, + return &TxReceipt{ + TxHash: receipt.TxHash.Bytes(), + ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, - BlockHash: receipt.BlockHash, + BlockHash: receipt.BlockHash.Bytes(), BlockNumber: receipt.BlockNumber, TransactionIndex: receipt.TransactionIndex, PostState: receipt.PostState, diff --git a/network/p2p/nethermind.go b/network/p2p/nethermind.go index a638deff..18f4c598 100644 --- a/network/p2p/nethermind.go +++ b/network/p2p/nethermind.go @@ -43,7 +43,7 @@ func (p *NethermindP2PProvider) EnrichStartCommand(bootnodes []string) []string } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { +func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -54,11 +54,11 @@ func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*pro logs = append(logs, *log) } - return &provide.TxReceipt{ - TxHash: receipt.TxHash, - ContractAddress: receipt.ContractAddress, + return &TxReceipt{ + TxHash: receipt.TxHash.Bytes(), + ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, - BlockHash: receipt.BlockHash, + BlockHash: receipt.BlockHash.Bytes(), BlockNumber: receipt.BlockNumber, TransactionIndex: receipt.TransactionIndex, PostState: receipt.PostState, diff --git a/network/p2p/parity.go b/network/p2p/parity.go index 73debac3..536cbed0 100644 --- a/network/p2p/parity.go +++ b/network/p2p/parity.go @@ -123,7 +123,7 @@ func (p *ParityP2PProvider) AddPeer(peerURL string) error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { +func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -134,11 +134,11 @@ func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide logs = append(logs, *log) } - return &provide.TxReceipt{ - TxHash: receipt.TxHash, - ContractAddress: receipt.ContractAddress, + return &TxReceipt{ + TxHash: receipt.TxHash.Bytes(), + ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, - BlockHash: receipt.BlockHash, + BlockHash: receipt.BlockHash.Bytes(), BlockNumber: receipt.BlockNumber, TransactionIndex: receipt.TransactionIndex, PostState: receipt.PostState, diff --git a/network/p2p/quorum.go b/network/p2p/quorum.go index d0c0535c..d33d1582 100644 --- a/network/p2p/quorum.go +++ b/network/p2p/quorum.go @@ -112,7 +112,7 @@ func (p *QuorumP2PProvider) DropNonReservedPeers() error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { +func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -123,11 +123,11 @@ func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide logs = append(logs, *log) } - return &provide.TxReceipt{ - TxHash: receipt.TxHash, - ContractAddress: receipt.ContractAddress, + return &TxReceipt{ + TxHash: receipt.TxHash.Bytes(), + ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, - BlockHash: receipt.BlockHash, + BlockHash: receipt.BlockHash.Bytes(), BlockNumber: receipt.BlockNumber, TransactionIndex: receipt.TransactionIndex, PostState: receipt.PostState, diff --git a/tx/shuttle_consumer.go b/tx/shuttle_consumer.go index de12186b..ab6c876a 100644 --- a/tx/shuttle_consumer.go +++ b/tx/shuttle_consumer.go @@ -198,6 +198,12 @@ func consumeShuttleContractDeployedMsg(msg *stan.Msg) { func(c *contract.Contract, tokenType, name string, decimals *big.Int, symbol string) (createdToken bool, tokenID uuid.UUID, errs []*provide.Error) { common.Log.Debugf("resolved %s token: %s (%v decimals); symbol: %s", *network.Name, name, decimals, symbol) + var address *string + if network.IsEthereumNetwork() { + ethCommonAddress := ethcommon.BytesToAddress(receipt.ContractAddress) + address = common.StringOrNil(ethCommonAddress.Hex()) + } + tok := &token.Token{ ApplicationID: c.ApplicationID, NetworkID: c.NetworkID, @@ -206,7 +212,7 @@ func consumeShuttleContractDeployedMsg(msg *stan.Msg) { Name: common.StringOrNil(name), Symbol: common.StringOrNil(symbol), Decimals: decimals.Uint64(), - Address: common.StringOrNil(receipt.ContractAddress.Hex()), + Address: address, } createdToken = tok.Create() diff --git a/tx/tx.go b/tx/tx.go index 35fe8089..955cbea0 100644 --- a/tx/tx.go +++ b/tx/tx.go @@ -26,6 +26,7 @@ import ( "github.com/provideplatform/nchain/common" "github.com/provideplatform/nchain/contract" "github.com/provideplatform/nchain/network" + "github.com/provideplatform/nchain/network/p2p" "github.com/provideplatform/nchain/token" "github.com/provideplatform/nchain/wallet" provide "github.com/provideplatform/provide-go/api" @@ -1343,12 +1344,17 @@ func (t *Transaction) handleTxReceipt( db *gorm.DB, network *network.Network, signerAddress string, - receipt *provideapi.TxReceipt, + receipt *p2p.TxReceipt, ) error { if t.To == nil { - common.Log.Debugf("Retrieved tx receipt for %s contract creation tx: %s; deployed contract address: %s", *network.Name, *t.Hash, receipt.ContractAddress.Hex()) + var contractAddress string + if network.IsEthereumNetwork() { + ethCommonAddress := ethcommon.BytesToAddress(receipt.ContractAddress) + contractAddress = *common.StringOrNil(ethCommonAddress.Hex()) + } + common.Log.Debugf("Retrieved tx receipt for %s contract creation tx: %s; deployed contract address: %s", *network.Name, *t.Hash, contractAddress) params := t.ParseParams() - contractName := fmt.Sprintf("Contract %s", *common.StringOrNil(receipt.ContractAddress.Hex())) + contractName := fmt.Sprintf("Contract %s", *common.StringOrNil(contractAddress)) if name, ok := params["name"].(string); ok { contractName = name } @@ -1367,7 +1373,7 @@ func (t *Transaction) handleTxReceipt( Name: common.StringOrNil(name), Symbol: common.StringOrNil(symbol), Decimals: decimals.Uint64(), - Address: common.StringOrNil(receipt.ContractAddress.Hex()), + Address: common.StringOrNil(contractAddress), } createdToken = tok.Create() @@ -1387,7 +1393,7 @@ func (t *Transaction) handleTxReceipt( NetworkID: t.NetworkID, TransactionID: &t.ID, Name: common.StringOrNil(contractName), - Address: common.StringOrNil(receipt.ContractAddress.Hex()), + Address: common.StringOrNil(contractAddress), Params: t.Params, Ref: t.Ref, } @@ -1399,7 +1405,7 @@ func (t *Transaction) handleTxReceipt( } } else { common.Log.Debugf("Using previously created contract %s for %s contract creation tx: %s, txID: %s", kontract.ID, *network.Name, *t.Hash, t.ID) - kontract.Address = common.StringOrNil(receipt.ContractAddress.Hex()) + kontract.Address = common.StringOrNil(contractAddress) db.Save(&kontract) common.Log.Debugf("Updated contract with address for txID: %s", t.ID) kontract.ResolveTokenContract(db, network, signerAddress, receipt, tokenCreateFn) @@ -1414,7 +1420,7 @@ func (t *Transaction) handleTxTraces( network *network.Network, signerAddress string, traces *provideapi.TxTrace, - receipt *provideapi.TxReceipt, + receipt *p2p.TxReceipt, ) error { kontract := t.GetContract(db) if kontract == nil || kontract.ID == uuid.Nil { @@ -1470,7 +1476,11 @@ func (t *Transaction) handleTxTraces( internalContract.ResolveTokenContract(db, network, signerAddress, receipt, func(c *contract.Contract, tokenType, name string, decimals *big.Int, symbol string) (createdToken bool, tokenID uuid.UUID, errs []*provide.Error) { common.Log.Debugf("Resolved %s token: %s (%v decimals); symbol: %s", *network.Name, name, decimals, symbol) - + var contractAddress *string + if network.IsEthereumNetwork() { + ethCommonAddress := ethcommon.BytesToAddress(receipt.ContractAddress) + contractAddress = common.StringOrNil(ethCommonAddress.Hex()) + } tok := &token.Token{ ApplicationID: c.ApplicationID, OrganizationID: c.OrganizationID, @@ -1480,7 +1490,7 @@ func (t *Transaction) handleTxTraces( Name: common.StringOrNil(name), Symbol: common.StringOrNil(symbol), Decimals: decimals.Uint64(), - Address: common.StringOrNil(receipt.ContractAddress.Hex()), + Address: contractAddress, } createdToken = tok.Create() From 1d8bb09cc8e7ea300aeaa887b68b0f0deb9393de Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Sat, 21 Aug 2021 16:37:06 +0200 Subject: [PATCH 05/15] Bump provide-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7501a45e..c3d63dc3 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/onsi/gomega v1.10.1 github.com/prometheus/tsdb v0.10.0 // indirect github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce - github.com/provideplatform/provide-go v0.0.0-20210701150626-bff9948fbd9a + github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1 github.com/spaolacci/murmur3 v1.1.1-0.20190317074736-539464a789e9 // indirect github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect go.mongodb.org/mongo-driver v1.5.1 diff --git a/go.sum b/go.sum index 1e4cbfbb..9e28c735 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,8 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce h1:0JZPWGnkBdFRGN/ptm49gMebcmMkLqnRukExazrvwCQ= github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce/go.mod h1:A0IzIOtqzS+MgpSTTKSQITY4kPH48IsGXw5pDCcegTU= github.com/provideplatform/provide-go v0.0.0-20210624064849-d7328258f0d8/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= -github.com/provideplatform/provide-go v0.0.0-20210701150626-bff9948fbd9a h1:9NekZCQm1lPSlh76Vu+mSlrr99ofxYMzdzltBCHcu9Q= -github.com/provideplatform/provide-go v0.0.0-20210701150626-bff9948fbd9a/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= +github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1 h1:oduICczibi9/+X6nHvsY+JOhyX1C7eVlg/19zt5O/AQ= +github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= github.com/provideservices/provide-go v0.0.0-20210409104111-70ad008e4ae8 h1:QaP1R8tsO69Pt5FQ2y8JZNpH5Dwk5xeNRcoethNcx3U= github.com/provideservices/provide-go v0.0.0-20210409104111-70ad008e4ae8/go.mod h1:zMvfECorm5+hb9aewWEVpxkaFGko3r5Qk3/N/aIuLeY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= From 7b080934717211c151d3e5a9784a9f54a816dff0 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Sat, 21 Aug 2021 16:38:22 +0200 Subject: [PATCH 06/15] Use types for provide-go and remove tmp structs --- cmd/statsdaemon/stats_daemon.go | 40 +------------ network/node.go | 4 +- network/p2p/baseledger.go | 101 ++------------------------------ network/p2p/common.go | 17 +----- network/p2p/fabric.go | 2 +- network/p2p/geth.go | 4 +- network/p2p/nethermind.go | 4 +- network/p2p/parity.go | 4 +- network/p2p/quorum.go | 4 +- tx/tx.go | 5 +- 10 files changed, 20 insertions(+), 165 deletions(-) diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index 8d49aee4..9fa734f6 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -333,42 +333,6 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta } } -// TMP struct, just for testing -// format is still unknown -type BaseledgerSubscriptionResponse struct { - ID interface{} `json:"id"` - Jsonrpc string `json:"jsonrpc"` - Result map[string]interface{} `json:"result"` -} - -// TMP struct, just for testing -// format is still unknown -type BaseledgerBlockHeader struct { - AppHash string `json:"app_hash"` - ChainID string `json:"chain_id"` - ConsensusHash string `json:"consensus_hash"` - DataHash string `json:"data_hash"` - EvidenceHash string `json:"evidence_hash"` - Height string `json:"height"` - LastBlockID struct { - Hash string `json:"hash"` - Parts struct { - Hash string `json:"hash"` - Total int `json:"total"` - } `json:"parts"` - } `json:"last_block_id"` - LastCommitHash string `json:"last_commit_hash"` - LastResultsHash string `json:"last_results_hash"` - NextValidatorsHash string `json:"next_validators_hash"` - ProposerAddress string `json:"proposer_address"` - Time time.Time `json:"time"` - ValidatorsHash string `json:"validators_hash"` - Version struct { - App string `json:"app"` - Block string `json:"block"` - } `json:"version"` -} - // BaseledgerNetworkStatsDataSourceFactory builds and returns a JSON-RPC and streaming websocket // data source which is used by stats daemon instances to consume EVM-based network statistics func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsDataSource { @@ -411,14 +375,14 @@ func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkS common.Log.Debugf("Received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) // TODO: try unmarshaling whole struct, for some reason parsing time is failing in header // even though tendermint timestamps layout is RFC3339 - response := &BaseledgerSubscriptionResponse{} + response := &provide.BaseledgerSubscriptionResponse{} err := json.Unmarshal(message, response) if err != nil { common.Log.Warningf("Failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) } else { if result, ok := response.Result["data"].(map[string]interface{}); ok { if resultJSON, err := json.Marshal(result); err == nil { - header := &BaseledgerBlockHeader{} + header := &provide.BaseledgerBlockHeader{} err := json.Unmarshal(resultJSON, header) if err != nil { common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) diff --git a/network/node.go b/network/node.go index 09054d76..93c722ec 100644 --- a/network/node.go +++ b/network/node.go @@ -735,6 +735,8 @@ func (n *Node) P2PAPIClient() (p2p.API, error) { var apiClient p2p.API switch client { + case p2p.ProviderBaseledger: + apiClient = p2p.InitBaseledgerP2PProvider(rpcURL, n.NetworkID.String(), n.Network) case p2p.ProviderBcoin: return nil, fmt.Errorf("Bcoin p2p provider not yet implemented") case p2p.ProviderGeth: @@ -749,8 +751,6 @@ func (n *Node) P2PAPIClient() (p2p.API, error) { apiClient = p2p.InitParityP2PProvider(rpcURL, n.NetworkID.String(), n.Network) case p2p.ProviderQuorum: apiClient = p2p.InitQuorumP2PProvider(rpcURL, n.NetworkID.String(), n.Network) - case p2p.ProviderBaseledger: - apiClient = p2p.InitBaseledgerP2PProvider(rpcURL, n.NetworkID.String(), n.Network) default: return nil, fmt.Errorf("Failed to resolve p2p provider for network node %s; unsupported client", n.ID) } diff --git a/network/p2p/baseledger.go b/network/p2p/baseledger.go index 348f7e27..9ba4b9ca 100644 --- a/network/p2p/baseledger.go +++ b/network/p2p/baseledger.go @@ -6,7 +6,6 @@ import ( "math/big" "strconv" "strings" - "time" "github.com/jinzhu/gorm" uuid "github.com/kthomas/go.uuid" @@ -61,7 +60,7 @@ func (p *BaseledgerP2PProvider) AddPeer(peerURL string) error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*nchain.TxReceipt, error) { httpClient := &provide.Client{ Host: *p.rpcURL, Scheme: "http", @@ -78,7 +77,7 @@ func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxR return nil, errors.New(string(respJSON)) } - txEntity := &TendermintTx{} + txEntity := &nchain.TendermintTx{} respJSON, _ := json.Marshal(resp) json.Unmarshal(respJSON, &txEntity) @@ -93,7 +92,7 @@ func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxR return nil, errors.New(string(respJSON)) } - blockEntity := &TendermintBlock{} + blockEntity := &nchain.TendermintBlock{} respJSON, _ = json.Marshal(resp) json.Unmarshal(respJSON, &blockEntity) @@ -102,7 +101,7 @@ func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxR blockNumber, _ := n.SetString(txEntity.Result.Height, 10) var logs []interface{} json.Unmarshal([]byte(txEntity.Result.TxResult.Log), &logs) - return &TxReceipt{ + return &nchain.TxReceipt{ TxHash: []byte(txEntity.Result.Hash), ContractAddress: nil, GasUsed: uint64(gasUsed), @@ -156,95 +155,3 @@ func (p *BaseledgerP2PProvider) RequireBootnodes(db *gorm.DB, userID *uuid.UUID, func (p *BaseledgerP2PProvider) Upgrade() error { return errors.New("not yet implemented") } - -// TODO: should move to provide-go models? -type TendermintTx struct { - Jsonrpc string `json:"jsonrpc"` - ID int `json:"id"` - Result struct { - Hash string `json:"hash"` - Height string `json:"height"` - Index int `json:"index"` - TxResult struct { - Code int `json:"code"` - Data string `json:"data"` - Log string `json:"log"` - Info string `json:"info"` - GasWanted string `json:"gas_wanted"` - GasUsed string `json:"gas_used"` - Events []struct { - Type string `json:"type"` - Attributes []struct { - Key string `json:"key"` - Value string `json:"value"` - Index bool `json:"index"` - } `json:"attributes"` - } `json:"events"` - Codespace string `json:"codespace"` - } `json:"tx_result"` - Tx string `json:"tx"` - } `json:"result"` -} - -type TendermintBlock struct { - Jsonrpc string `json:"jsonrpc"` - ID int `json:"id"` - Result struct { - BlockID struct { - Hash string `json:"hash"` - Parts struct { - Total int `json:"total"` - Hash string `json:"hash"` - } `json:"parts"` - } `json:"block_id"` - Block struct { - Header struct { - Version struct { - Block string `json:"block"` - } `json:"version"` - ChainID string `json:"chain_id"` - Height string `json:"height"` - Time time.Time `json:"time"` - LastBlockID struct { - Hash string `json:"hash"` - Parts struct { - Total int `json:"total"` - Hash string `json:"hash"` - } `json:"parts"` - } `json:"last_block_id"` - LastCommitHash string `json:"last_commit_hash"` - DataHash string `json:"data_hash"` - ValidatorsHash string `json:"validators_hash"` - NextValidatorsHash string `json:"next_validators_hash"` - ConsensusHash string `json:"consensus_hash"` - AppHash string `json:"app_hash"` - LastResultsHash string `json:"last_results_hash"` - EvidenceHash string `json:"evidence_hash"` - ProposerAddress string `json:"proposer_address"` - } `json:"header"` - Data struct { - Txs []string `json:"txs"` - } `json:"data"` - Evidence struct { - Evidence []interface{} `json:"evidence"` - } `json:"evidence"` - LastCommit struct { - Height string `json:"height"` - Round int `json:"round"` - BlockID struct { - Hash string `json:"hash"` - Parts struct { - Total int `json:"total"` - Hash string `json:"hash"` - } `json:"parts"` - } `json:"block_id"` - Signatures []struct { - BlockIDFlag int `json:"block_id_flag"` - ValidatorAddress string `json:"validator_address"` - Timestamp time.Time `json:"timestamp"` - Signature string `json:"signature"` - } `json:"signatures"` - } `json:"last_commit"` - } `json:"block"` - } `json:"result"` -} diff --git a/network/p2p/common.go b/network/p2p/common.go index f72520d7..01bfbf1d 100644 --- a/network/p2p/common.go +++ b/network/p2p/common.go @@ -71,7 +71,7 @@ type API interface { AddPeer(string) error RemovePeer(string) error ParsePeerURL(string) (*string, error) - FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) + FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) FetchTxTraces(hash string) (*provide.TxTrace, error) FormatBootnodes([]string) string RequireBootnodes(db *gorm.DB, userID *uuid.UUID, networkID *uuid.UUID, n common.Configurable) error @@ -83,21 +83,6 @@ type API interface { EnrichStartCommand(bootnodes []string) []string } -// move to provide-go -type TxReceipt struct { - TxHash []byte `json:"hash"` - ContractAddress []byte `json:"contract_address"` - GasUsed uint64 `json:"gas_used"` - BlockHash []byte `json:"block_hash,omitempty"` - BlockNumber *big.Int `json:"block,omitempty"` - TransactionIndex uint `json:"transaction_index"` - PostState []byte `json:"root"` - Status uint64 `json:"status"` - CumulativeGasUsed uint64 `json:"cumulative_gas_used"` - Bloom interface{} `json:"logs_bloom"` - Logs []interface{} `json:"logs"` -} - func evmFetchTxReceipt(rpcClientKey, rpcURL, signerAddress, hash string) (*types.Receipt, error) { receipt, err := providecrypto.EVMGetTxReceipt(rpcClientKey, rpcURL, hash, signerAddress) if err != nil { diff --git a/network/p2p/fabric.go b/network/p2p/fabric.go index 23ac8acf..0fa633d7 100644 --- a/network/p2p/fabric.go +++ b/network/p2p/fabric.go @@ -49,7 +49,7 @@ func (p *HyperledgerFabricP2PProvider) EnrichStartCommand(bootnodes []string) [] } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *HyperledgerFabricP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { return nil, errors.New("fabric does not impl FetchTxReceipt()") } diff --git a/network/p2p/geth.go b/network/p2p/geth.go index 6ba469de..f8ea9781 100644 --- a/network/p2p/geth.go +++ b/network/p2p/geth.go @@ -95,7 +95,7 @@ func (p *GethP2PProvider) EnrichStartCommand(bootnodes []string) []string { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -106,7 +106,7 @@ func (p *GethP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt logs = append(logs, *log) } - return &TxReceipt{ + return &provide.TxReceipt{ TxHash: receipt.TxHash.Bytes(), ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, diff --git a/network/p2p/nethermind.go b/network/p2p/nethermind.go index 18f4c598..ab880cc1 100644 --- a/network/p2p/nethermind.go +++ b/network/p2p/nethermind.go @@ -43,7 +43,7 @@ func (p *NethermindP2PProvider) EnrichStartCommand(bootnodes []string) []string } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -54,7 +54,7 @@ func (p *NethermindP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxR logs = append(logs, *log) } - return &TxReceipt{ + return &provide.TxReceipt{ TxHash: receipt.TxHash.Bytes(), ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, diff --git a/network/p2p/parity.go b/network/p2p/parity.go index 536cbed0..7739ef4e 100644 --- a/network/p2p/parity.go +++ b/network/p2p/parity.go @@ -123,7 +123,7 @@ func (p *ParityP2PProvider) AddPeer(peerURL string) error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -134,7 +134,7 @@ func (p *ParityP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxRecei logs = append(logs, *log) } - return &TxReceipt{ + return &provide.TxReceipt{ TxHash: receipt.TxHash.Bytes(), ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, diff --git a/network/p2p/quorum.go b/network/p2p/quorum.go index d33d1582..d205ef6d 100644 --- a/network/p2p/quorum.go +++ b/network/p2p/quorum.go @@ -112,7 +112,7 @@ func (p *QuorumP2PProvider) DropNonReservedPeers() error { } // FetchTxReceipt fetch a transaction receipt given its hash -func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxReceipt, error) { +func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*provide.TxReceipt, error) { receipt, err := evmFetchTxReceipt(p.networkID, *p.rpcURL, signerAddress, hash) if err != nil { return nil, err @@ -123,7 +123,7 @@ func (p *QuorumP2PProvider) FetchTxReceipt(signerAddress, hash string) (*TxRecei logs = append(logs, *log) } - return &TxReceipt{ + return &provide.TxReceipt{ TxHash: receipt.TxHash.Bytes(), ContractAddress: receipt.ContractAddress.Bytes(), GasUsed: receipt.GasUsed, diff --git a/tx/tx.go b/tx/tx.go index 955cbea0..c9a630e8 100644 --- a/tx/tx.go +++ b/tx/tx.go @@ -26,7 +26,6 @@ import ( "github.com/provideplatform/nchain/common" "github.com/provideplatform/nchain/contract" "github.com/provideplatform/nchain/network" - "github.com/provideplatform/nchain/network/p2p" "github.com/provideplatform/nchain/token" "github.com/provideplatform/nchain/wallet" provide "github.com/provideplatform/provide-go/api" @@ -1344,7 +1343,7 @@ func (t *Transaction) handleTxReceipt( db *gorm.DB, network *network.Network, signerAddress string, - receipt *p2p.TxReceipt, + receipt *provideapi.TxReceipt, ) error { if t.To == nil { var contractAddress string @@ -1420,7 +1419,7 @@ func (t *Transaction) handleTxTraces( network *network.Network, signerAddress string, traces *provideapi.TxTrace, - receipt *p2p.TxReceipt, + receipt *provideapi.TxReceipt, ) error { kontract := t.GetContract(db) if kontract == nil || kontract.ID == uuid.Nil { From 2866b166f35199d0e82873b4f149bc194afad2b0 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Sat, 21 Aug 2021 18:02:51 +0200 Subject: [PATCH 07/15] Format tx receipt hashes --- network/p2p/baseledger.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/network/p2p/baseledger.go b/network/p2p/baseledger.go index 9ba4b9ca..0153cc87 100644 --- a/network/p2p/baseledger.go +++ b/network/p2p/baseledger.go @@ -3,6 +3,7 @@ package p2p import ( "encoding/json" "errors" + "fmt" "math/big" "strconv" "strings" @@ -102,10 +103,10 @@ func (p *BaseledgerP2PProvider) FetchTxReceipt(signerAddress, hash string) (*nch var logs []interface{} json.Unmarshal([]byte(txEntity.Result.TxResult.Log), &logs) return &nchain.TxReceipt{ - TxHash: []byte(txEntity.Result.Hash), + TxHash: []byte(fmt.Sprintf("0x%s", string(txEntity.Result.Hash))), ContractAddress: nil, GasUsed: uint64(gasUsed), - BlockHash: []byte(blockEntity.Result.BlockID.Hash), + BlockHash: []byte(fmt.Sprintf("0x%s", string(blockEntity.Result.BlockID.Hash))), BlockNumber: blockNumber, TransactionIndex: 0, PostState: nil, From 2809ac878a9a3485a9d25c7018eaadfccf817866 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Sun, 22 Aug 2021 19:44:35 +0200 Subject: [PATCH 08/15] Fix unmarshalling format --- cmd/statsdaemon/stats_daemon.go | 50 +++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index 9fa734f6..c8e832af 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -333,6 +333,51 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta } } +type HeaderResponse struct { + Type string `json:"type"` + Value struct { + Header struct { + AppHash string `json:"app_hash"` + ChainID string `json:"chain_id"` + ConsensusHash string `json:"consensus_hash"` + DataHash string `json:"data_hash"` + EvidenceHash string `json:"evidence_hash"` + Height string `json:"height"` + LastBlockID struct { + Hash string `json:"hash"` + Parts struct { + Hash string `json:"hash"` + Total int `json:"total"` + } `json:"parts"` + } `json:"last_block_id"` + LastCommitHash string `json:"last_commit_hash"` + LastResultsHash string `json:"last_results_hash"` + NextValidatorsHash string `json:"next_validators_hash"` + ProposerAddress string `json:"proposer_address"` + Time time.Time `json:"time"` + ValidatorsHash string `json:"validators_hash"` + Version struct { + App string `json:"app"` + Block string `json:"block"` + } `json:"version"` + } `json:"header"` + NumTxs string `json:"num_txs"` + ResultBeginBlock struct { + Events []struct { + Attributes []struct { + Index bool `json:"index"` + Key string `json:"key"` + Value string `json:"value"` + } `json:"attributes"` + Type string `json:"type"` + } `json:"events"` + } `json:"result_begin_block"` + ResultEndBlock struct { + ValidatorUpdates interface{} `json:"validator_updates"` + } `json:"result_end_block"` + } `json:"value"` +} + // BaseledgerNetworkStatsDataSourceFactory builds and returns a JSON-RPC and streaming websocket // data source which is used by stats daemon instances to consume EVM-based network statistics func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsDataSource { @@ -373,8 +418,6 @@ func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkS break } else { common.Log.Debugf("Received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) - // TODO: try unmarshaling whole struct, for some reason parsing time is failing in header - // even though tendermint timestamps layout is RFC3339 response := &provide.BaseledgerSubscriptionResponse{} err := json.Unmarshal(message, response) if err != nil { @@ -382,11 +425,12 @@ func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkS } else { if result, ok := response.Result["data"].(map[string]interface{}); ok { if resultJSON, err := json.Marshal(result); err == nil { - header := &provide.BaseledgerBlockHeader{} + header := &HeaderResponse{} err := json.Unmarshal(resultJSON, header) if err != nil { common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) } else if header != nil { + common.Log.Debugf("Block header received %v\n", result) ch <- &provide.NetworkStatus{ Meta: map[string]interface{}{ "last_block_header": result, From 130b6769249be9d8c031bae620a16b840ce680b3 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Sun, 22 Aug 2021 20:43:17 +0200 Subject: [PATCH 09/15] Bump provide-go and use proper header response --- cmd/statsdaemon/stats_daemon.go | 56 ++++----------------------------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 9 insertions(+), 53 deletions(-) diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index c8e832af..8bf69de8 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -26,6 +26,7 @@ import ( uuid "github.com/kthomas/go.uuid" "github.com/provideplatform/nchain/common" "github.com/provideplatform/nchain/network" + "github.com/provideplatform/provide-go/api/nchain" provide "github.com/provideplatform/provide-go/api/nchain" providecrypto "github.com/provideplatform/provide-go/crypto" ) @@ -333,51 +334,6 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta } } -type HeaderResponse struct { - Type string `json:"type"` - Value struct { - Header struct { - AppHash string `json:"app_hash"` - ChainID string `json:"chain_id"` - ConsensusHash string `json:"consensus_hash"` - DataHash string `json:"data_hash"` - EvidenceHash string `json:"evidence_hash"` - Height string `json:"height"` - LastBlockID struct { - Hash string `json:"hash"` - Parts struct { - Hash string `json:"hash"` - Total int `json:"total"` - } `json:"parts"` - } `json:"last_block_id"` - LastCommitHash string `json:"last_commit_hash"` - LastResultsHash string `json:"last_results_hash"` - NextValidatorsHash string `json:"next_validators_hash"` - ProposerAddress string `json:"proposer_address"` - Time time.Time `json:"time"` - ValidatorsHash string `json:"validators_hash"` - Version struct { - App string `json:"app"` - Block string `json:"block"` - } `json:"version"` - } `json:"header"` - NumTxs string `json:"num_txs"` - ResultBeginBlock struct { - Events []struct { - Attributes []struct { - Index bool `json:"index"` - Key string `json:"key"` - Value string `json:"value"` - } `json:"attributes"` - Type string `json:"type"` - } `json:"events"` - } `json:"result_begin_block"` - ResultEndBlock struct { - ValidatorUpdates interface{} `json:"validator_updates"` - } `json:"result_end_block"` - } `json:"value"` -} - // BaseledgerNetworkStatsDataSourceFactory builds and returns a JSON-RPC and streaming websocket // data source which is used by stats daemon instances to consume EVM-based network statistics func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsDataSource { @@ -425,15 +381,15 @@ func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkS } else { if result, ok := response.Result["data"].(map[string]interface{}); ok { if resultJSON, err := json.Marshal(result); err == nil { - header := &HeaderResponse{} - err := json.Unmarshal(resultJSON, header) + headerResponse := &nchain.BaseledgerBlockHeaderResponse{} + err := json.Unmarshal(resultJSON, headerResponse) if err != nil { common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) - } else if header != nil { - common.Log.Debugf("Block header received %v\n", result) + } else if headerResponse != nil { + common.Log.Debugf("Block header received %v\n", headerResponse.Value.Header) ch <- &provide.NetworkStatus{ Meta: map[string]interface{}{ - "last_block_header": result, + "last_block_header": headerResponse.Value.Header, }, } } diff --git a/go.mod b/go.mod index c3d63dc3..d6897f41 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/onsi/gomega v1.10.1 github.com/prometheus/tsdb v0.10.0 // indirect github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce - github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1 + github.com/provideplatform/provide-go v0.0.0-20210822183020-1c080cdc4a77 github.com/spaolacci/murmur3 v1.1.1-0.20190317074736-539464a789e9 // indirect github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect go.mongodb.org/mongo-driver v1.5.1 diff --git a/go.sum b/go.sum index 9e28c735..fa2eb0fa 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,8 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce h1:0JZPWGnkBdFRGN/ptm49gMebcmMkLqnRukExazrvwCQ= github.com/provideplatform/ident v0.9.10-0.20210727215953-cdd9940805ce/go.mod h1:A0IzIOtqzS+MgpSTTKSQITY4kPH48IsGXw5pDCcegTU= github.com/provideplatform/provide-go v0.0.0-20210624064849-d7328258f0d8/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= -github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1 h1:oduICczibi9/+X6nHvsY+JOhyX1C7eVlg/19zt5O/AQ= -github.com/provideplatform/provide-go v0.0.0-20210820142615-4728f8d567a1/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= +github.com/provideplatform/provide-go v0.0.0-20210822183020-1c080cdc4a77 h1:+/LcHOVu85ySEFunR/6f8k4GcBBb/xug86pGHkohc6I= +github.com/provideplatform/provide-go v0.0.0-20210822183020-1c080cdc4a77/go.mod h1:q0/Q8KaZxYg84rdwBIIE7ZwHluzM5zw7zJJoJOqAbzg= github.com/provideservices/provide-go v0.0.0-20210409104111-70ad008e4ae8 h1:QaP1R8tsO69Pt5FQ2y8JZNpH5Dwk5xeNRcoethNcx3U= github.com/provideservices/provide-go v0.0.0-20210409104111-70ad008e4ae8/go.mod h1:zMvfECorm5+hb9aewWEVpxkaFGko3r5Qk3/N/aIuLeY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= From 12b08affda1d9b548897700ce36270f20e330062 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Tue, 24 Aug 2021 02:39:59 +0200 Subject: [PATCH 10/15] Add first version of stats daemon test --- cmd/statsdaemon/main_test.go | 124 ++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/cmd/statsdaemon/main_test.go b/cmd/statsdaemon/main_test.go index 72190280..7d17fc9b 100644 --- a/cmd/statsdaemon/main_test.go +++ b/cmd/statsdaemon/main_test.go @@ -1,15 +1,137 @@ package main import ( + "encoding/json" + "fmt" "testing" + uuid "github.com/kthomas/go.uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/provideplatform/ident/common" + "github.com/provideplatform/nchain/network" + ident "github.com/provideplatform/provide-go/api/ident" + "github.com/provideplatform/provide-go/api/nchain" ) -func TestNChainStatsdaemon(t *testing.T) { +type chainSpecConfig struct{} + +type chainSpec struct { + Config *chainSpecConfig `json:"config"` +} + +type chainConfig struct { + NativeCurrency *string `json:"native_currency"` + IsBaseledgerNetwork bool `json:"is_baseledger_network"` + Client *string `json:"client"` + BlockExplorerUrl *string `json:"block_explorer_url"` + JsonRpcUrl *string `json:"json_rpc_url"` + WebsocketUrl *string `json:"websocket_url"` + Platform *string `json:"platform"` + EngineID *string `json:"engine_id"` + Chain *string `json:"chain"` + ProtocolID *string `json:"protocol_id"` + NetworkID int `json:"network_id"` + ChainSpec *chainSpec `json:"chainspec"` +} + +type chainDef struct { + Name *string `json:"name"` + Enabled bool `json:"enabled"` + Config *chainConfig `json:"config"` +} + +func SetupBaseledgerTestNetwork() (*network.Network, error) { + testID, _ := uuid.NewV4() + + email := "prvd" + testID.String() + "@email.com" + pwd := "super_secret" + _, err := ident.CreateUser("", map[string]interface{}{ + "first_name": "statsdaemon first name" + testID.String(), + "last_name": "statsdaemon last name" + testID.String(), + "email": email, + "password": pwd, + }) + if err != nil { + return nil, fmt.Errorf("error creating user. Error: %s", err.Error()) + } + + authResponse, _ := ident.Authenticate(email, pwd) + if err != nil { + return nil, fmt.Errorf("error authenticating user. Error: %s", err.Error()) + } + + chainySpecConfig := chainSpecConfig{} + chainySpec := chainSpec{ + Config: &chainySpecConfig, + } + chainyConfig := chainConfig{ + NativeCurrency: common.StringOrNil("token"), + Platform: common.StringOrNil("tendermint"), + Client: common.StringOrNil("baseledger"), + NetworkID: 3, + IsBaseledgerNetwork: true, + Chain: common.StringOrNil("peachtree"), + WebsocketUrl: common.StringOrNil("ws://genesis.peachtree.baseledger.provide.network:1337/websocket"), + ChainSpec: &chainySpec, + } + + chainName := fmt.Sprintf("Baseledger Testnet %s", testID.String()) + + chainyChain := chainDef{ + Name: common.StringOrNil(chainName), + Config: &chainyConfig, + } + + chainyChainJSON, _ := json.Marshal(chainyChain) + + params := map[string]interface{}{} + json.Unmarshal(chainyChainJSON, ¶ms) + + testNetwork, err := nchain.CreateNetwork(*authResponse.Token.AccessToken, params) + if err != nil { + return nil, fmt.Errorf("error creating network. Error: %s", err.Error()) + } + + return &network.Network{ + ApplicationID: testNetwork.ApplicationID, + UserID: testNetwork.UserID, + Name: testNetwork.Name, + Description: testNetwork.Description, + Enabled: testNetwork.Enabled, + ChainID: testNetwork.ChainID, + NetworkID: testNetwork.NetworkID, + Config: testNetwork.Config, + }, nil +} + +func TestNChainBaseledgerStatsdaemon(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "NChain statsdaemon Suite") + + testNetwork, err := SetupBaseledgerTestNetwork() + if err != nil { + t.Errorf("Failed to set up baseledger test network %v", err.Error()) + } + + statsDaemon := NewNetworkStatsDaemon(common.Log, testNetwork) + + testCh := make(chan *nchain.NetworkStatus) + // go routine since this is blocking + go statsDaemon.dataSource.Stream(testCh) + // get one result and shutdown statsdaemon and check result + sampleResult := <-testCh + statsDaemon.shutdown() + + jsonSampleResult, _ := json.Marshal(sampleResult.Meta["last_block_header"]) + formattedSampleHeaderResult := nchain.BaseledgerBlockHeaderResponse{}.Value.Header + err = json.Unmarshal(jsonSampleResult, &formattedSampleHeaderResult) + if err != nil { + t.Errorf("Failed to unmarshall header response %v", err.Error()) + } + + t.Logf("Header response success %v\n", formattedSampleHeaderResult) } var _ = Describe("Main", func() { From eb9b40105a60765ad0542b2c0d7b7717e0531ac8 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Tue, 24 Aug 2021 02:57:52 +0200 Subject: [PATCH 11/15] Remove not needed test channel and use stats daemon queue --- cmd/statsdaemon/main_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/statsdaemon/main_test.go b/cmd/statsdaemon/main_test.go index 7d17fc9b..29778f72 100644 --- a/cmd/statsdaemon/main_test.go +++ b/cmd/statsdaemon/main_test.go @@ -115,13 +115,9 @@ func TestNChainBaseledgerStatsdaemon(t *testing.T) { t.Errorf("Failed to set up baseledger test network %v", err.Error()) } - statsDaemon := NewNetworkStatsDaemon(common.Log, testNetwork) - - testCh := make(chan *nchain.NetworkStatus) - // go routine since this is blocking - go statsDaemon.dataSource.Stream(testCh) + statsDaemon := RequireNetworkStatsDaemon(testNetwork) // get one result and shutdown statsdaemon and check result - sampleResult := <-testCh + sampleResult := <-statsDaemon.queue statsDaemon.shutdown() jsonSampleResult, _ := json.Marshal(sampleResult.Meta["last_block_header"]) From 710c794ce14c38f19475d7136ebc15dca059579f Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Tue, 24 Aug 2021 03:16:39 +0200 Subject: [PATCH 12/15] Utilize gingko framework --- cmd/statsdaemon/main_test.go | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/statsdaemon/main_test.go b/cmd/statsdaemon/main_test.go index 29778f72..04f74143 100644 --- a/cmd/statsdaemon/main_test.go +++ b/cmd/statsdaemon/main_test.go @@ -109,27 +109,27 @@ func SetupBaseledgerTestNetwork() (*network.Network, error) { func TestNChainBaseledgerStatsdaemon(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "NChain statsdaemon Suite") - - testNetwork, err := SetupBaseledgerTestNetwork() - if err != nil { - t.Errorf("Failed to set up baseledger test network %v", err.Error()) - } - - statsDaemon := RequireNetworkStatsDaemon(testNetwork) - // get one result and shutdown statsdaemon and check result - sampleResult := <-statsDaemon.queue - statsDaemon.shutdown() - - jsonSampleResult, _ := json.Marshal(sampleResult.Meta["last_block_header"]) - formattedSampleHeaderResult := nchain.BaseledgerBlockHeaderResponse{}.Value.Header - err = json.Unmarshal(jsonSampleResult, &formattedSampleHeaderResult) - if err != nil { - t.Errorf("Failed to unmarshall header response %v", err.Error()) - } - - t.Logf("Header response success %v\n", formattedSampleHeaderResult) } var _ = Describe("Main", func() { - + It("Should parse one successful baseledger block header event", func() { + testNetwork, err := SetupBaseledgerTestNetwork() + if err != nil { + Fail("Failed to set up baseledger test network") + } + + statsDaemon := RequireNetworkStatsDaemon(testNetwork) + // get one result and shutdown statsdaemon and check result + sampleResult := <-statsDaemon.queue + statsDaemon.shutdown() + + jsonSampleResult, _ := json.Marshal(sampleResult.Meta["last_block_header"]) + formattedSampleHeaderResult := nchain.BaseledgerBlockHeaderResponse{}.Value.Header + err = json.Unmarshal(jsonSampleResult, &formattedSampleHeaderResult) + if err != nil { + Fail("Failed to unmarshall header response") + } + + Expect(formattedSampleHeaderResult).NotTo(BeNil()) + }) }) From dd4e7e8946a66ab79a39b7f9f171850b71ea7194 Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Tue, 24 Aug 2021 23:57:43 +0200 Subject: [PATCH 13/15] Use evict daemon method instead of just shutting it down --- cmd/statsdaemon/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/statsdaemon/main_test.go b/cmd/statsdaemon/main_test.go index 04f74143..6623ab94 100644 --- a/cmd/statsdaemon/main_test.go +++ b/cmd/statsdaemon/main_test.go @@ -121,7 +121,7 @@ var _ = Describe("Main", func() { statsDaemon := RequireNetworkStatsDaemon(testNetwork) // get one result and shutdown statsdaemon and check result sampleResult := <-statsDaemon.queue - statsDaemon.shutdown() + EvictNetworkStatsDaemon(testNetwork) jsonSampleResult, _ := json.Marshal(sampleResult.Meta["last_block_header"]) formattedSampleHeaderResult := nchain.BaseledgerBlockHeaderResponse{}.Value.Header From 086b0dcc21dc4717f2f825ce62a969ab69a22d5f Mon Sep 17 00:00:00 2001 From: Stefan Kostic Date: Wed, 25 Aug 2021 00:40:22 +0200 Subject: [PATCH 14/15] Add separate scripts for running statsdaemon tests --- ops/run_integration_tests_statsdaemon.sh | 8 + ops/run_local_tests_statsdaemon.sh | 308 +++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 ops/run_integration_tests_statsdaemon.sh create mode 100644 ops/run_local_tests_statsdaemon.sh diff --git a/ops/run_integration_tests_statsdaemon.sh b/ops/run_integration_tests_statsdaemon.sh new file mode 100644 index 00000000..37ef2eee --- /dev/null +++ b/ops/run_integration_tests_statsdaemon.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +docker volume prune -f +docker build -t nchain/local --no-cache . +docker-compose -f ./ops/docker-compose.yml up -d +TAGS=$LOCAL_TAGS ./ops/run_local_tests_statsdaemon.sh +docker-compose -f ./ops/docker-compose.yml down +docker volume rm ops_provide-db diff --git a/ops/run_local_tests_statsdaemon.sh b/ops/run_local_tests_statsdaemon.sh new file mode 100644 index 00000000..42874c01 --- /dev/null +++ b/ops/run_local_tests_statsdaemon.sh @@ -0,0 +1,308 @@ +#!/bin/bash + +set -e +echo "" > coverage.txt + +if [[ -z "${DATABASE_NAME}" ]]; then + DATABASE_NAME=nchain_dev +fi + +if [[ -z "${DATABASE_USER}" ]]; then + DATABASE_USER=nchain +fi + +if [[ -z "${DATABASE_PASSWORD}" ]]; then + DATABASE_PASSWORD=nchain +fi + +if [[ -z "${NATS_SERVER_PORT}" ]]; then + NATS_SERVER_PORT=4221 +fi + +if [[ -z "${NATS_STREAMING_SERVER_PORT}" ]]; then + NATS_STREAMING_SERVER_PORT=4222 +fi + +if [[ -z "${NATS_STREAMING_CONCURRENCY}" ]]; then + NATS_STREAMING_CONCURRENCY=1 +fi + +if [[ -z "${NATS_CONCURRENCY}" ]]; then + NATS_CONCURRENCY=1 +fi + +if [[ -z "${REDIS_SERVER_PORT}" ]]; then + REDIS_SERVER_PORT=6379 +fi + +if [[ -z "${RACE}" ]]; then + RACE=true +fi + +if [[ -z "${TAGS}" ]]; then + TAGS=unit +fi + +#dropdb $DATABASE_NAME || true >/dev/null +#dropuser $DATABASE_USER || true >/dev/null + +#PGPASSWORD=$DATABASE_SUPERUSER_PASSWORD dropdb -U $DATABASE_SUPERUSER -h 0.0.0.0 -p $DATABASE_PORT $DATABASE_NAME || true >/dev/null +#PGPASSWORD=$DATABASE_SUPERUSER_PASSWORD dropuser -U $DATABASE_SUPERUSER -h 0.0.0.0 -p $DATABASE_PORT $DATABASE_USER || true >/dev/null + +DATABASE_USER=$DATABASE_USER DATABASE_PASSWORD=$DATABASE_PASSWORD DATABASE_NAME=$DATABASE_NAME make migrate + +PGP_PUBLIC_KEY='-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF1Db+IBEAC0nRf3s6rls6jhWeWWTAJY8Nn4+qPUbSu0ZOx1DAqOHHxYAek1 +TOuogsXaFPRtRL5mO+0aRIDjqo6GKp9IC8k6XFlJ/+LU1C09O5XOkbzhVoHtTHOY +dvLY1N3Pw5tzemFnbjMVrbTcuLgVAZoW9+e1GTUJT/VUL6AVYhg51U3r8sOuiUX5 +wJrpGF4dhtOUc6pv3aBuG/iqA7vrJ8lME/3kdUZIMcs+StqJBxBuk/GykPAp5de9 +vofqVd8h1aZKBjHCcdDvGDK2bLqyVk+0lE8zoh/2HG+52y/dqdVt6VEsRuf96Cou +pGeftbXKkgHv4pf0ySrNoXr3bkZmuf+SJyfF+hBq34G4zGVdT3IYH5Dwsd+ScQ72 +KVI9XuO3sny+TUSWIXjWFTpQ0mhtjMhdHngXERBcgmdaS5JfmgGev3l0tqOBWhXA +oObRQ8oPhWhF20sM7LHWZSqbf4GGiVShCK6RxlRm2Uwhl1Fjx+1ThtKkq+JUgPrs +hCtk0CZVXlKIbJjrvRhJ7x/fjDEyfXur4wscHrGJr45M+3ts1dRhKUxKpNl8k9p1 +RXEEntNcsV0FAZz4B0l7ImGVOKK9mdlcRLVMZ37NC5QeEoOcglHB3wGpMMmXcZU1 +ZlRkQt0M/FE/PU4pKXtqiyGUZP/EFzY2O+adZZCXAbgmdbC+ktsnQCWpCwARAQAB +tCpwcm92aWRlLWRldiA8ZW5naW5lZXJpbmdAcHJvdmlkZS5zZXJ2aWNlcz6JAk4E +EwEIADgWIQS8yJWwHRuPxwaoDKhhrI+FPBEK6AUCXUNv4gIbAwULCQgHAgYVCgkI +CwIEFgIDAQIeAQIXgAAKCRBhrI+FPBEK6MHrEACm8uJ2Xc20vnXmJCuMqL3KsR75 +JKcGJw8G6z3EpRjV8FeZTfOpjO/joe+X7HrCgKq8RTfnoduApEYY7Jut5Cqlw9VF +ImQMfYUBOjMzrfbkBMngjd4P7FcFAOL+amgYP85whSoKZL1EdJkpiScM16i4rvAv +LHC8BzLS8XrkF52uMV4uaFDlgI1VVhm/Q0U/9g/WJBHXXugEpnuttT3TT0rD8BLY +bIxRdsli8M0N+c2BBfISA5kNUl2j7MEqhKPuDXWHHRBkDxKwrk/mROjJWexOtUTl +GR+WFPs2W4ikhhX51zPUCYnrhm/WFrjy+xwNveaNGrk4pr3Hm6YEGnTD5H71t4vW +ezlpbfA7aPLq6/HX30JsLUpFyl/PTE4BAAizuu04HbnRHlNBsITa7pbf2MMN4tSe +3uxcHql8BDmm3RSSK2N1vwMszySPmapUVKi3W6qduTBgT9WCd77mt20259Lyurqu +s7UF0CvrHfxGeyrMkXbv13xwG+wqWpI2hmiOTkEg2zhxap16sWhQVNJP/r1sqqUa +BwQ8cGI62hEwd9rF49etznl4TmzqUvQCx+VEnbArT/7nUKHmGPXAcI7g0hMJj0N6 +LQXF5w8xlHFXOfzpDNN7YzobARgiWC41NezPZKhv4y8Cy2riUsju5qRyIN37Ipy1 +rE2pI5SJ+jU80Z+G2bkCDQRdQ2/iARAArsi1Zuqv9raZWLLcAgyk1ed6KtmvWxTH +/zf6E37P2dh+Z/7pGJp6xkHUdtgaj7EABEZfzGJnyo11Pna7u8se0j1ZtxwazjS+ +B48KW9eM0K9m8eje0RkP99jg3naZGV1LnJKt2g68h5rsibfBJYg7/tYxBpqILavW +CCg9xTDsN0X9D5hQdg2pclG9OsSmK72rBbzuEkKnhY99U+2LluCIi5bX07gYuZ9G +lbWtX6IKlvPx/FDLaOqUWrtJ0WYQMb92TqimbRe6MG2vLzPIKrn8MkaSSADd9nhT +rnKD61hr4kFr+sd+iyr1nKg97ppd4dWaL88oDEeKE9xO3H391ihz0stbt2AfD9R0 +2i86UfY06lLfEukiqlwUGnvxVntjDZmqJWZBG7mLtilSC1THpifPMJ2OogSJRMJ9 +wZVelI2aK6uSrqkzW8dutKhcRmlRVDCtMdpFJncxuyLneVWKBMH8SMVenYrbrkqq +1NN/25H4Wy7A/4MCvMHoBRMQheiFBEWx+JEkWnDFeVA2ZGREmQsqncG81GN/pULW +shlCEgMU3LrKoB2wzfgBjVXjKq/+7utuwKshkEH2y6g5IPDOOHtfdXNqZ3Ph3r0l +k/KqWY56csJ0ytUXVYpcr1u7VxS/yoOQQBkh481CSJfV7eRJ5/epEQKdxP4sPFCv +00SZv1/XyvkAEQEAAYkCNgQYAQgAIBYhBLzIlbAdG4/HBqgMqGGsj4U8EQroBQJd +Q2/iAhsMAAoJEGGsj4U8EQroTusP/jKLeLOaqXWIlZqSWR5kaYUWeEvtZWVWQLsa +kqKEIFwTSq5HBWIfyZxtPTmo3JN+78ufkYJXg0Hs/sTIWbs3juoDSgLvql2ME1M2 +a3Dgr2OPkMPnEReQ8jlcRcseJqqdl2CpFG6pQzmof8UZZ3G//GHIop6Fss29+hjg +fDAOc/YXwcrqVs0cPpXSBU+XJwH7zIvkVfRKFeoieOqK8mHhgDP6FTJ2tyR8+UDa +NsaB7rF2LJyh4UtJLImlftH/1yYWjaYxjsOqiqiTA5NPbiEKtfnK54740YVJouAq +CCoQV6vwMbvre3izUSCPcmHWdo1Lpy8MV1vrDto7biptubqM6Sli0BUlP6/EhFnY +DaKMm9iZvsS6q5mHIcmZxuqMkJ6gheMISqN4tzoGshOpE+XPRk6GgJp6GxTsEuim +MBjvqsdVq8lqp1kGWdSZDgnOPOPwvW3mIp8nEwY7b8hp/YGz6pylaQaCKeACQkfd +r82Hnp5/o36qMdudanGGvP28qbXmEan/VyKQuReBJ2JQKLUpFpCWUOaho6dcP6Zo +jNfRxN+DUEJNER0oZUTEeEno3BfRYkpQ/EZjtQ9muVh2S8UVL06OV0f5deOxicP4 +65KorIgQeczCg8iX3Pt1ZojYNW4YOnrEys22ZaI6+hmvLf9Zx4u4ip+tTIaEkJoi +96kKxfZ6 +=HwI4 +-----END PGP PUBLIC KEY BLOCK-----' + +PGP_PRIVATE_KEY='-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQdGBF1Db+IBEAC0nRf3s6rls6jhWeWWTAJY8Nn4+qPUbSu0ZOx1DAqOHHxYAek1 +TOuogsXaFPRtRL5mO+0aRIDjqo6GKp9IC8k6XFlJ/+LU1C09O5XOkbzhVoHtTHOY +dvLY1N3Pw5tzemFnbjMVrbTcuLgVAZoW9+e1GTUJT/VUL6AVYhg51U3r8sOuiUX5 +wJrpGF4dhtOUc6pv3aBuG/iqA7vrJ8lME/3kdUZIMcs+StqJBxBuk/GykPAp5de9 +vofqVd8h1aZKBjHCcdDvGDK2bLqyVk+0lE8zoh/2HG+52y/dqdVt6VEsRuf96Cou +pGeftbXKkgHv4pf0ySrNoXr3bkZmuf+SJyfF+hBq34G4zGVdT3IYH5Dwsd+ScQ72 +KVI9XuO3sny+TUSWIXjWFTpQ0mhtjMhdHngXERBcgmdaS5JfmgGev3l0tqOBWhXA +oObRQ8oPhWhF20sM7LHWZSqbf4GGiVShCK6RxlRm2Uwhl1Fjx+1ThtKkq+JUgPrs +hCtk0CZVXlKIbJjrvRhJ7x/fjDEyfXur4wscHrGJr45M+3ts1dRhKUxKpNl8k9p1 +RXEEntNcsV0FAZz4B0l7ImGVOKK9mdlcRLVMZ37NC5QeEoOcglHB3wGpMMmXcZU1 +ZlRkQt0M/FE/PU4pKXtqiyGUZP/EFzY2O+adZZCXAbgmdbC+ktsnQCWpCwARAQAB +/gcDAnQZdqwxtBcf6c4goheF+3hZirCCimY2u8A8vRU89kH0evSv7yyHMikCBb+o +lf3l+iWRcPDnwYfCCdVsou0ED1o+5CZGOlzo7ZZ6xdYTzLOthMhva8lxeADXh0j1 +Pm/VPj9sl8CU7ghBK4wb9gkBBUqBUJ5Mdk+pSkmcK5Xrh8dkcseN5LF7KpQfGSO6 +1BjiRQj56uLwjwcnqLABvlp83cBkXYOQAMr3Fk8GaFVccNkvBqfGO7U3Fk5vxrZR +x2iZubwNMp0Erie2mj3hZqNtP4nB+iXylraNObgjTQBLYUfAsJBSg0Kg8IkqZOMM +dEJanl8w6gLi+19kk8rZpYHAdulaWT2rCRd5tAJfLjJ/amSyJYG99zofDM7CJrcV +5/3TdP/5yOoRlVOMpd2Mnu8W4G8sNJHNZPLlc6WoyHESldTGnWVODuIN5lcSuh1F +Xg1DE05utT+kDruk3GT5CWnTm5Hq8GLUFltGJ0aoob5PYjT6b6/+ZPB0vZQO3jzD +aUHjpU6bN9vwKqxbusM+94mLX1/W01CLZxVHU5mSj/iUG6iMsQdZN2xluFfzsJ2g +8t3izfMqPdoF3LFTyAjsBcZK+UOr8DkBk9wJEJAGUDZtMYZXx59cAYuG7w1dkcKI +yMRl3K0zIc+OZMUC+83xkxFpoEp2xcV/UOx9LgFsGyVnqe/MMmWsslNVopOWLL8l +7oaHN1UAujHQ3r3+inJqEpLqScu/p/xapVjsWE76vCFhGtXmpe7AzH59bzqNMuOD +WuAC3tNJeKI8/gVLrx4AHqhCw1PaY35OXL8LU4JdoEp0wB5oedJLCDac7obY0ILu +kFhJg7DtZZMSsKIshfqc0Yy3Ij6wlN2EbTO0iGDRaORJ1Sp6d3N46NfLVUfmR5MW +CsTxvvY9JAdX2DtVxGgwGkpO9irZ3G/gucnORtVweCxM24yxbgdlv8g9DQtmcvyS +/dCVNLGYbu0dNOYtxIwx0QGzqwiJKoOs2fiUKe7IPz0t1RyD/SpE/5NaanD5Ii8s +1/DiZonc4FILVUoLGoEy6uE6FSKAgULg3MgE4c19/DBSTC5Y1sACsBbyKObW4uS6 +Oxmi6X43mBuEa/1G18+0Yv6+5JlAE+EggpLqR/oIO+i0Wsg0O+l4Jmv8KPxm/FPg +vJYjKuKCGtXnPGARRNOWueQjk9RA2HG9kSvKUklI1wN6ucthS4nbGsurWS+zwtHV +6GHbjDX+RoXJTkHHIznpMGGI6E38cIe1NjxLlcWKajXtSyHXxDHkmaNxphSn9BSt +QV6TzmV/vQDDqXpUXbWjubtlfJzrikic/Sg71b1gvcPuLvLYptZ9P+9NxPx9/8ID +eMkLdHb76+6Y4va/mBLYJEqDqq7L7/6ceY/gGcUHEhH64/ItEzhnLpI03Ycp90zN +zbOv0QSZwrxBPkT2KEtYAfmIgm0U0as8fsqBE8u2okCFAkPRDSKN2vQbIkzCEj1I +Ejk9phH8qqi8R0Ti+SJLr6AFD/LR3oVV1YbgsSknjR8YB6qmeH/zNAKY+1kwOVLv +PuMWMVWzgYqrmO9Ho9ibYwVNzh9FvetvDQImLAz/jFGsst+K8EXdBuq1r3MpEhrF +IE4Nzu0/gioIfAxcgZjJDWJAn4ILwAyb+kSD1bafmmhaMyKqwoqPLbSfXc88G5Hx +ba1bBFC0njZ7qfB11/1vvQ96Z+wKyK8So3eQU2prAe8S3VN6VV52OLPAa0hg3vH5 +wGEbwRJjzu0xn/7sSvxTER8bU/Wiw0h0y+BH7tpYgJBNPRLVv93TvH20KnByb3Zp +ZGUtZGV2IDxlbmdpbmVlcmluZ0Bwcm92aWRlLnNlcnZpY2VzPokCTgQTAQgAOBYh +BLzIlbAdG4/HBqgMqGGsj4U8EQroBQJdQ2/iAhsDBQsJCAcCBhUKCQgLAgQWAgMB +Ah4BAheAAAoJEGGsj4U8EQrowesQAKby4nZdzbS+deYkK4yovcqxHvkkpwYnDwbr +PcSlGNXwV5lN86mM7+Oh75fsesKAqrxFN+eh24CkRhjsm63kKqXD1UUiZAx9hQE6 +MzOt9uQEyeCN3g/sVwUA4v5qaBg/znCFKgpkvUR0mSmJJwzXqLiu8C8scLwHMtLx +euQXna4xXi5oUOWAjVVWGb9DRT/2D9YkEdde6ASme621PdNPSsPwEthsjFF2yWLw +zQ35zYEF8hIDmQ1SXaPswSqEo+4NdYcdEGQPErCuT+ZE6MlZ7E61ROUZH5YU+zZb +iKSGFfnXM9QJieuGb9YWuPL7HA295o0auTimvcebpgQadMPkfvW3i9Z7OWlt8Dto +8urr8dffQmwtSkXKX89MTgEACLO67TgdudEeU0GwhNrult/Yww3i1J7e7FweqXwE +OabdFJIrY3W/AyzPJI+ZqlRUqLdbqp25MGBP1YJ3vua3bTbn0vK6uq6ztQXQK+sd +/EZ7KsyRdu/XfHAb7CpakjaGaI5OQSDbOHFqnXqxaFBU0k/+vWyqpRoHBDxwYjra +ETB32sXj163OeXhObOpS9ALH5USdsCtP/udQoeYY9cBwjuDSEwmPQ3otBcXnDzGU +cVc5/OkM03tjOhsBGCJYLjU17M9kqG/jLwLLauJSyO7mpHIg3fsinLWsTakjlIn6 +NTzRn4bZnQdGBF1Db+IBEACuyLVm6q/2tplYstwCDKTV53oq2a9bFMf/N/oTfs/Z +2H5n/ukYmnrGQdR22BqPsQAERl/MYmfKjXU+dru7yx7SPVm3HBrONL4Hjwpb14zQ +r2bx6N7RGQ/32ODedpkZXUuckq3aDryHmuyJt8EliDv+1jEGmogtq9YIKD3FMOw3 +Rf0PmFB2DalyUb06xKYrvasFvO4SQqeFj31T7YuW4IiLltfTuBi5n0aVta1fogqW +8/H8UMto6pRau0nRZhAxv3ZOqKZtF7owba8vM8gqufwyRpJIAN32eFOucoPrWGvi +QWv6x36LKvWcqD3uml3h1ZovzygMR4oT3E7cff3WKHPSy1u3YB8P1HTaLzpR9jTq +Ut8S6SKqXBQae/FWe2MNmaolZkEbuYu2KVILVMemJ88wnY6iBIlEwn3BlV6UjZor +q5KuqTNbx260qFxGaVFUMK0x2kUmdzG7Iud5VYoEwfxIxV6dituuSqrU03/bkfhb +LsD/gwK8wegFExCF6IUERbH4kSRacMV5UDZkZESZCyqdwbzUY3+lQtayGUISAxTc +usqgHbDN+AGNVeMqr/7u627AqyGQQfbLqDkg8M44e191c2pnc+HevSWT8qpZjnpy +wnTK1RdVilyvW7tXFL/Kg5BAGSHjzUJIl9Xt5Enn96kRAp3E/iw8UK/TRJm/X9fK ++QARAQAB/gcDAp/NuTo7+BdI6YiI8RRw2uy8ZoyKJT2D76uX9/U10Ej2MLlfWfED +h+s6M+9q3rWcLctwZ4NHgowcT+CgJ8muxcxbpfhjHtWHOipl1YArUJQCoW1Fiwyy +aktQM4KBudAm2+TwcxetSRxn6YyAZLMs0j/Ax/7/Q0pLhqpmodV42CcOXhmhMLRn +/MpHt19HfopE1RrUXlgr0jA8gtiz6vi10j93tKNgL7Va7rjgx4NbJB/MIQzI+9GW +U4xb9eOBLpv2JD65PsjJZaqfGsiJPOwBtHXBTtt/9a4nskaX442DqLXvzp6750Ot +stD2tV8ONMfiR/C4UlVijOfL4yT579AXnuMZvpYenSuTMDHmcLkARUBK/xt50C45 +AWXGXvsUXKTDrljtixaUlj8ADMeTRan3ZjoiAGWfNFAzBPzhoq+vSBsWDxE8U5zH +P+pQJEHm935HbZ30kK6FYo5BXr1ak8enSq1UGGZN6JSoHcL7KQdQPmJPxNpc7GVA +KUNrpwvOaQx39N9dfO6p4FpGhAw//YLBlssmvK/rST6Hg98nzQcflpoAI4SPXYPH +Y5TO75PpYvH5InlKNf4B0rMpvblJRnIGQ9LTHrqRZWAh6DnB2kXsHVEm3FX1tQC7 +0zMtrS5cgIFgC8v0OWLJpgV3y76szvfoWiAVq6ltIf0FG74Hgxm9Eldzc467aSKx +0u0GA0xa6FYjIwdts5dEUk00yKjzhx7aDEo7PZ2ipFfU0vDSD0sq94EZ+yqJN+LZ +xlH5s/JV1wbthoxABOGA3fVJu61O6vYFj3W63USwRgTgpvZz8+LnsligEiksC2rL +GP8WDZ96Hjl/0atS/yciDI2yyscEF/eLMqVq65Yecfx1VPxNL2f/e2g5TpudAa4l +YRexENyRagtn1jsT13qLVSK253n5GEFySTpih18gjHX2OUzOShDlg0m5smbnE1aN +eUswuIOVfX0T2GqleFNlQ0PD5DkHbR1WHIZzahi9QT8Rzqqnk7zguaejLUayaHjw +SGZ68AHP6whOlU1pbjtRlKINeKm3G9UXZfjs51EYflpeaqUUB/XWYDjcRO6P0J3w +Ul1YIp2geD5+F2gVc8HLiHD5PZThI69r2ERv/f/aS4XVezchOln/cb/eyPZGw04v ++N1NUzOhitn5vpRJGKFtioRfVXMSlYxszBAp6JUQvkEMO3rznTlnR0XzPuCrD/Zv +TsCIC1X34S+5q0A0PvZivHBwJo7zsMu7JJF1/ESQ9KOmLUpBVnETXv2ATfRP5tBs +GQ5LilYbQ8SDiGKfOo6eP1tOgksBRkXu+tXPBONn8hvPJllSycK7JqTXSckVqZvT +b/APmWPa4VyxY5oMjRnCq5vskhvd7Wi9ig1R1pJwe2jixb6fy8vCxt/lUfkkm1oZ +u5Orm6+QQKlpt2N4wZJzVuoWDkCLD4qPnILRbh5MHOx0UuXZ2S+dUkTf+O07fpjt +TsNavKHkTjQ83tGuVKZz49ozO8/QwgkZrfgmRrJgHHI38HnMyShKN9bFzVKnjHaq +dRCTj5HNpkPtzqY4V2ueacRtqVp3rZ037dUJ28jYKFatHyBm11p3PSpB2kqVLm5A +lBva7JyIpliDUZqe6EKU26PJL2/ThLz8qx435aStMjEw2YLH6UXcblk6B7CjkpfR +wBTWl8F4BLq2XaC/lmepG/OFzl7W3dYhNh6obToX3b5yS72xG75Q6U+CJnvBH5QX +q8tZ1LDtO1UIAqCYFfmdY9ywmIxIb0iD7R/ZZ5HBKxj1fU2OMVIQo3Rt5AlDGEiJ +AjYEGAEIACAWIQS8yJWwHRuPxwaoDKhhrI+FPBEK6AUCXUNv4gIbDAAKCRBhrI+F +PBEK6E7rD/4yi3izmql1iJWaklkeZGmFFnhL7WVlVkC7GpKihCBcE0quRwViH8mc +bT05qNyTfu/Ln5GCV4NB7P7EyFm7N47qA0oC76pdjBNTNmtw4K9jj5DD5xEXkPI5 +XEXLHiaqnZdgqRRuqUM5qH/FGWdxv/xhyKKehbLNvfoY4HwwDnP2F8HK6lbNHD6V +0gVPlycB+8yL5FX0ShXqInjqivJh4YAz+hUydrckfPlA2jbGge6xdiycoeFLSSyJ +pX7R/9cmFo2mMY7DqoqokwOTT24hCrX5yueO+NGFSaLgKggqEFer8DG763t4s1Eg +j3Jh1naNS6cvDFdb6w7aO24qbbm6jOkpYtAVJT+vxIRZ2A2ijJvYmb7EuquZhyHJ +mcbqjJCeoIXjCEqjeLc6BrITqRPlz0ZOhoCaehsU7BLopjAY76rHVavJaqdZBlnU +mQ4Jzjzj8L1t5iKfJxMGO2/Iaf2Bs+qcpWkGgingAkJH3a/Nh56ef6N+qjHbnWpx +hrz9vKm15hGp/1cikLkXgSdiUCi1KRaQllDmoaOnXD+maIzX0cTfg1BCTREdKGVE +xHhJ6NwX0WJKUPxGY7UPZrlYdkvFFS9OjldH+XXjsYnD+OuSqKyIEHnMwoPIl9z7 +dWaI2DVuGDp6xMrNtmWiOvoZry3/WceLuIqfrUyGhJCaIvepCsX2eg== +=1A2t +-----END PGP PRIVATE KEY BLOCK-----' + +PGP_PASSPHRASE=password + +PAYMENTS_REFRESH_TOKEN='eyJhbGciOiJSUzI1NiIsImtpZCI6ImU2OmY3OmQ1OjI0OmUyOjU5OjA2OjJiOmJjOmEyOjhjOjM1OjlkOmNhOjBhOjg3IiwidHlwIjoiSldUIn0.eyJhdWQiOiJodHRwczovL2lkZW50LnByb3ZpZGUuc2VydmljZXMvYXBpL3YxIiwiaWF0IjoxNjAwNzA5NzU0LCJpc3MiOiJodHRwczovL2lkZW50LnByb3ZpZGUuc2VydmljZXMiLCJqdGkiOiJjN2I2YzI2ZS00OTkwLTQ4YWYtYmMwYy05YWRiY2E5ZmRmNzYiLCJuYXRzIjp7InBlcm1pc3Npb25zIjp7InN1YnNjcmliZSI6eyJhbGxvdyI6WyJhcHBsaWNhdGlvbi4wMTU1NGUyMi0zZDdhLTQ0YTMtOWM2NS02YmNhYmFhMDhjMzgiLCJuZXR3b3JrLiouY29ubmVjdG9yLioiLCJuZXR3b3JrLiouc3RhdHVzIiwicGxhdGZvcm0uXHUwMDNlIl19fX0sInBydmQiOnsiYXBwbGljYXRpb25faWQiOiIwMTU1NGUyMi0zZDdhLTQ0YTMtOWM2NS02YmNhYmFhMDhjMzgiLCJleHRlbmRlZCI6eyJwZXJtaXNzaW9ucyI6eyIqIjo1MTB9fSwicGVybWlzc2lvbnMiOjUxMH0sInN1YiI6ImFwcGxpY2F0aW9uOjAxNTU0ZTIyLTNkN2EtNDRhMy05YzY1LTZiY2FiYWEwOGMzOCJ9.iPYYSS0hHNYLUXcgpBfQbo6goMGDHF5Oxv1OvkB-WAzRgZSAm2HFroOUsmPlCQwO5eNeTfMqRaQMDdl6idTCip99y-zYTu8ys7dahyk4P1lhh4BB8vTCl3AHQuyUTGloMrY2JytpkmXMZTsxu-UhQxaaQN0IlSotSIFAYPT3jHH5nYy2MJbcfxePt8xKmXzwvpjTEVJRmUfAfEXjJF34S3hAuw9S7WncKucZfuP1WwP65h53HbLB69DR6KFZ76eiRavke5RpT40r9UKC6zPP-UZhTAuWQjOSmBhkd_IUg4T2a8r4W9CJT6aLgtwE0i1OUrPDVj_EzQV9tsjlwIOv5y9r_p-sfdxXdHFfoT8nAs5uIcWTw45J2Ycc0b4vqs-sYDr2qn7TS5DvJbPQSnRBS9YZ8CJq9mFpc5GjunCzEqO6JkvEWaN1mqPJbcvMGmLRQt5zA-2D0fFq1mvIUCUcg3EQ5J5lAZqudGf9mnYf4xRIMacCssF5VsP36xXg7pnscqh3u3JdQ-Fon3nB5vbIXn2fxaJjYl4ggNr-IgLxK7_h9KlDkiv7I7EKWGl2Np0q3-mVvuTIk7M-GqT3Dx9TtpR6MsK6EX0frUH3bZH8RHBHnxx67oxNMamviT-XUNudUU7Wan1PfnaPSsqfrn6OT5Abep-BbewKJn3ErY0Z-oU' +NCHAIN_API_HOST='localhost:8080' +NCHAIN_API_PATH='api/v1' +NCHAIN_API_SCHEME='http' +IDENT_API_HOST='localhost:8081' +IDENT_API_PATH='api/v1' +IDENT_API_SCHEME='http' +VAULT_API_HOST='localhost:8082' +VAULT_API_PATH='api/v1' +VAULT_API_SCHEME='http' +VAULT_SEAL_UNSEAL_KEY='traffic charge swing glimpse will citizen push mutual embrace volcano siege identify gossip battle casual exit enrich unlock muscle vast female initial please day' +VAULT_REFRESH_TOKEN='eyJhbGciOiJSUzI1NiIsImtpZCI6IjEwOjJlOmQ5OmUxOmI4OmEyOjM0OjM3Ojk5OjNhOjI0OmZjOmFhOmQxOmM4OjU5IiwidHlwIjoiSldUIn0.eyJhdWQiOiJodHRwczovL3Byb3ZpZGUuc2VydmljZXMvYXBpL3YxIiwiaWF0IjoxNjA1NzkxMjQ4LCJpc3MiOiJodHRwczovL2lkZW50LnByb3ZpZGUuc2VydmljZXMiLCJqdGkiOiI5YjUxNGIxNS01NTdlLTRhYWQtYTcwOC0wMTcwZTAwZWE1ZmIiLCJuYXRzIjp7InBlcm1pc3Npb25zIjp7InN1YnNjcmliZSI6eyJhbGxvdyI6WyJhcHBsaWNhdGlvbi4zNjAxNTdmOC1kNWExLTQ0NDAtOTE4Yi1mNjhiYjM5YzBkODAiLCJ1c2VyLjIzY2MwN2UwLTM4NTEtNDBkZC1iNjc1LWRmNzY4MDY3MmY3ZCIsIm5ldHdvcmsuKi5jb25uZWN0b3IuKiIsIm5ldHdvcmsuKi5zdGF0dXMiLCJwbGF0Zm9ybS5cdTAwM2UiXX19fSwicHJ2ZCI6eyJhcHBsaWNhdGlvbl9pZCI6IjM2MDE1N2Y4LWQ1YTEtNDQ0MC05MThiLWY2OGJiMzljMGQ4MCIsImV4dGVuZGVkIjp7InBlcm1pc3Npb25zIjp7IioiOjUxMH19LCJwZXJtaXNzaW9ucyI6NTEwLCJ1c2VyX2lkIjoiMjNjYzA3ZTAtMzg1MS00MGRkLWI2NzUtZGY3NjgwNjcyZjdkIn0sInN1YiI6ImFwcGxpY2F0aW9uOjM2MDE1N2Y4LWQ1YTEtNDQ0MC05MThiLWY2OGJiMzljMGQ4MCJ9.SUh84MKBNstdu3KFu1zEAQq03xbPw1D0lLXeogz1HfBJy77bIGf7HLvCuc6bjkh0xj3cEuEus1dC1Dj3BvlZoSXsvz_biTzSapkXzJjpkwOL6qkYDmqTPZvXwqmk-mUNrHTPkqdiIJL7xA46tzHW3E_hjSA9HjEk1kXjPdJQ6_ifkgWNoAaSD--kudIrhZ7vLnfy0H1JEAOsXzSAMoc5_pNG2n79m0ywvb_4l9BqdsHW8N3xSQOFjcp9gD_tqo6ffug3pkpoy-RSguM_OaMR2lj_CHhYxAt0phtjUceDD3K1h5iZ38kSl7izhOdULMmGBhVpBMoSy6_R6ZzpCL3pj8FcReX9RXR5oYpm8PDtlmWqblQzjwY00-uYLfOX0_iS4MGfEsjadZPfTmJLcOTYC7H4PL9ZRu_XtMDUrGBQQz5b_ad2ZzMXbBNeU6vbxVKDG8VFKWOHAemqHTcvuOAsOCLIqOu-eJpZHlXbx-FXPTYledd-GBDe7IjaC9ll_JK3utCOnCq0qUs6lnXIrQ_Sp1LcTKJJ7aY5f9TxeoAuL-ghDbQ3Xkw6huKyPCz2evOwVLwrB9ZRMlQXgmTnB1OeQvWii1WbmkyV1Zhbz_RPB8ckK7_mFxuPvsXK8wTFiWFmj96sRX470kV-ooSfM5CzKZhSLqgyyaUNC0VaCPq0uuE' + +#pkgs=(bridge common connector consumer contract db filter network oracle prices token tx wallet) +pkgs=("cmd/statsdaemon") + +for d in "${pkgs[@]}" ; do + pkg=$(echo $d | sed 's/\/*$//g') + + if [ "$RACE" = "true" ]; then + PGP_PUBLIC_KEY=${PGP_PUBLIC_KEY} \ + PGP_PRIVATE_KEY=$PGP_PRIVATE_KEY \ + PGP_PASSPHRASE=$PGP_PASSPHRASE \ + NATS_TOKEN=testtoken \ + NATS_URL=nats://localhost:${NATS_SERVER_PORT} \ + NATS_STREAMING_URL=nats://localhost:${NATS_STREAMING_SERVER_PORT} \ + NATS_CLUSTER_ID=provide \ + NATS_STREAMING_URL=$NATS_STREAMING_URL \ + NATS_STREAMING_CONCURRENCY=$NATS_STREAMING_CONCURRENCY \ + NATS_CONCURRENCY=$NATS_CONCURRENCY \ + GIN_MODE=release \ + DATABASE_HOST=localhost \ + DATABASE_NAME=${DATABASE_NAME} \ + DATABASE_USER=${DATABASE_USER} \ + DATABASE_PASSWORD=${DATABASE_PASSWORD} \ + LOG_LEVEL=DEBUG \ + REDIS_HOSTS=localhost:${REDIS_SERVER_PORT} \ + TEST_AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID} \ + TEST_AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY} \ + PAYMENTS_REFRESH_TOKEN=${PAYMENTS_REFRESH_TOKEN} \ + IDENT_API_HOST=${IDENT_API_HOST} \ + IDENT_API_SCHEME=${IDENT_API_SCHEME} \ + VAULT_API_HOST=${VAULT_API_HOST} \ + VAULT_API_SCHEME=${VAULT_API_SCHEME} \ + NCHAIN_API_HOST=${NCHAIN_API_HOST} \ + NCHAIN_API_SCHEME=http \ + VAULT_SEAL_UNSEAL_KEY=${VAULT_SEAL_UNSEAL_KEY} \ + VAULT_REFRESH_TOKEN=${VAULT_REFRESH_TOKEN} \ + go test "./${pkg}" -v \ + -race \ + -timeout 1800s \ + -cover \ + -coverpkg="./${pkg}" \ + -coverprofile=profile.out \ + -tags="$TAGS" + else + PGP_PUBLIC_KEY=$PGP_PUBLIC_KEY \ + PGP_PRIVATE_KEY=$PGP_PRIVATE_KEY \ + PGP_PASSPHRASE=$PGP_PASSPHRASE \ + NATS_TOKEN=testtoken \ + NATS_URL=nats://localhost:${NATS_SERVER_PORT} \ + NATS_STREAMING_URL=nats://localhost:${NATS_STREAMING_SERVER_PORT} \ + NATS_CLUSTER_ID=provide \ + NATS_STREAMING_URL=$NATS_STREAMING_URL \ + NATS_STREAMING_CONCURRENCY=$NATS_STREAMING_CONCURRENCY \ + NATS_CONCURRENCY=$NATS_CONCURRENCY \ + GIN_MODE=release \ + DATABASE_HOST=localhost \ + DATABASE_NAME=${DATABASE_NAME} \ + DATABASE_USER=${DATABASE_USER} \ + DATABASE_PASSWORD=${DATABASE_PASSWORD} \ + LOG_LEVEL=DEBUG \ + REDIS_HOSTS=localhost:${REDIS_SERVER_PORT} \ + TEST_AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID} \ + TEST_AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY} \ + PAYMENTS_REFRESH_TOKEN=${PAYMENTS_REFRESH_TOKEN} \ + IDENT_API_HOST=${IDENT_API_HOST} \ + IDENT_API_SCHEME=${IDENT_API_SCHEME} \ + #VAULT_API_HOST=${VAULT_API_HOST} \ + #VAULT_API_SCHEME=${VAULT_API_SCHEME} \ + #NCHAIN_API_HOST=${NCHAIN_API_HOST} \ + #NCHAIN_API_SCHEME=${NCHAIN_API_SCHEME} \ + #VAULT_REFRESH_TOKEN=${VAULT_REFRESH_TOKEN} \ + #VAULT_SEAL_UNSEAL_KEY=${VAULT_SEAL_UNSEAL_KEY} \ + go test "./${pkg}" -v \ + -timeout 1800s \ + -cover \ + -coverpkg="./${pkg}" \ + -coverprofile=profile.out \ + -tags="$TAGS" + fi +done From e0e8fabcbe20323aa958e8973223ce4d7b628b03 Mon Sep 17 00:00:00 2001 From: Kyle Thomas Date: Wed, 25 Aug 2021 23:46:43 -0400 Subject: [PATCH 15/15] Cleanup logging in statsdaemon and tx packages --- cmd/statsdaemon/log_transceiver.go | 38 ++++---- cmd/statsdaemon/stats_daemon.go | 135 +++++++++++++---------------- network/consumer.go | 50 +++++------ tx/consumer.go | 120 ++++++++++++------------- tx/shuttle_consumer.go | 28 +++--- 5 files changed, 175 insertions(+), 196 deletions(-) diff --git a/cmd/statsdaemon/log_transceiver.go b/cmd/statsdaemon/log_transceiver.go index dee6eed4..423f2dba 100644 --- a/cmd/statsdaemon/log_transceiver.go +++ b/cmd/statsdaemon/log_transceiver.go @@ -65,7 +65,7 @@ func EthereumLogTransceiverFactory(network *network.Network) *LogTransceiver { var wsDialer websocket.Dialer wsConn, _, err := wsDialer.Dial(websocketURL, nil) if err != nil { - common.Log.Errorf("Failed to establish network logs websocket connection to %s; %s", websocketURL, err.Error()) + common.Log.Errorf("failed to establish network logs websocket connection to %s; %s", websocketURL, err.Error()) } else { defer wsConn.Close() id, _ := uuid.NewV4() @@ -77,9 +77,9 @@ func EthereumLogTransceiverFactory(network *network.Network) *LogTransceiver { "jsonrpc": "2.0", } if err := wsConn.WriteJSON(payload); err != nil { - common.Log.Errorf("Failed to write subscribe message to network logs websocket connection") + common.Log.Errorf("failed to write subscribe message to network logs websocket connection") } else { - common.Log.Debugf("Subscribed to network logs websocket: %s", websocketURL) + common.Log.Debugf("subscribed to network logs websocket: %s", websocketURL) for { _, message, err := wsConn.ReadMessage() @@ -116,13 +116,13 @@ func EthereumLogTransceiverFactory(network *network.Network) *LogTransceiver { // Consume the websocket stream; attempts to fallback to JSON-RPC if websocket stream fails or is not available for the network func (lt *LogTransceiver) consume() error { - lt.log.Debugf("Attempting to consume configured network log transceiver; attempt #%v", lt.attempt) + lt.log.Debugf("attempting to consume configured network log transceiver; attempt #%v", lt.attempt) var err error if lt.Stream != nil { err = lt.Stream(lt.queue) } else { - err = errors.New("Configured log transceiver does not have a configured Stream impl") + err = errors.New("configured log transceiver does not have a configured Stream impl") } return err @@ -131,7 +131,7 @@ func (lt *LogTransceiver) consume() error { func (lt *LogTransceiver) ingest(logmsg []byte) { defer func() { if r := recover(); r != nil { - common.Log.Warningf("Recovered from failed log transceiver event ingestion attempt; %s", r) + common.Log.Warningf("recovered from failed log transceiver event ingestion attempt; %s", r) } }() @@ -143,7 +143,7 @@ func (lt *LogTransceiver) ingest(logmsg []byte) { func (lt *LogTransceiver) ingestEthereum(logmsg []byte) { err := natsutil.NatsStreamingPublish(natsLogTransceiverEmitSubject, logmsg) if err != nil { - common.Log.Warningf("Log transceiver failed to publish %d-byte log emission message; %s", len(logmsg), err.Error()) + common.Log.Warningf("log transceiver failed to publish %d-byte log emission message; %s", len(logmsg), err.Error()) } } @@ -155,7 +155,7 @@ func (lt *LogTransceiver) loop() error { lt.ingest(*evt) case <-lt.shutdownCtx.Done(): - lt.log.Debugf("Closing log transceiver on shutdown") + lt.log.Debugf("closing log transceiver on shutdown") return nil } } @@ -164,14 +164,14 @@ func (lt *LogTransceiver) loop() error { // EvictNetworkLogTransceiver evicts a single, previously-initialized log transceiver instance { func EvictNetworkLogTransceiver(network *network.Network) error { if daemon, ok := currentLogTransceivers[network.ID.String()]; ok { - common.Log.Debugf("Evicting log transceiver instance for network: %s; id: %s", *network.Name, network.ID) + common.Log.Debugf("evicting log transceiver instance for network: %s; id: %s", *network.Name, network.ID) daemon.shutdown() currentLogTransceiversMutex.Lock() delete(currentLogTransceivers, network.ID.String()) currentLogTransceiversMutex.Unlock() return nil } - return fmt.Errorf("Unable to evict log transceiver instance for network: %s; id; %s", *network.Name, network.ID) + return fmt.Errorf("unable to evict log transceiver instance for network: %s; id; %s", *network.Name, network.ID) } // RequireNetworkLogTransceiver ensures a single log transceiver instance is running for @@ -180,12 +180,12 @@ func EvictNetworkLogTransceiver(network *network.Network) error { func RequireNetworkLogTransceiver(network *network.Network) *LogTransceiver { var daemon *LogTransceiver if daemon, ok := currentLogTransceivers[network.ID.String()]; ok { - common.Log.Debugf("Cached log transceiver instance found for network: %s; id: %s", *network.Name, network.ID) + common.Log.Debugf("cached log transceiver instance found for network: %s; id: %s", *network.Name, network.ID) return daemon } currentLogTransceiversMutex.Lock() - common.Log.Infof("Initializing new log transceiver instance for network: %s; id: %s", *network.Name, network.ID) + common.Log.Infof("initializing new log transceiver instance for network: %s; id: %s", *network.Name, network.ID) daemon = NewNetworkLogTransceiver(common.Log, network) if daemon != nil { currentLogTransceivers[network.ID.String()] = daemon @@ -219,10 +219,10 @@ func (lt *LogTransceiver) run() error { go func() { for !lt.shuttingDown() { lt.attempt++ - common.Log.Debugf("Stepping into main runloop of log transceiver instance; attempt #%v", lt.attempt) + common.Log.Debugf("stepping into main runloop of log transceiver instance; attempt #%v", lt.attempt) err := lt.consume() if err != nil { - common.Log.Warningf("Configured network log transceiver failed to consume network log events; %s", err.Error()) + common.Log.Warningf("configured network log transceiver failed to consume network log events; %s", err.Error()) if lt.backoff == 0 { lt.backoff = 100 } else { @@ -240,10 +240,10 @@ func (lt *LogTransceiver) run() error { err := lt.loop() if err == nil { - lt.log.Info("Network log transceiver exited cleanly") + lt.log.Info("network log transceiver exited cleanly") } else { if !lt.shuttingDown() { - common.Log.Errorf("Forcing shutdown of log transceiver due to error; %s", err) + common.Log.Errorf("forcing shutdown of log transceiver due to error; %s", err) lt.shutdown() } } @@ -252,14 +252,14 @@ func (lt *LogTransceiver) run() error { } func (lt *LogTransceiver) handleSignals() { - common.Log.Debug("Installing SIGINT and SIGTERM signal handlers") + common.Log.Debug("installing SIGINT and SIGTERM signal handlers") sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { select { case sig := <-sigs: - common.Log.Infof("Received signal: %s", sig) + common.Log.Infof("received signal: %s", sig) lt.shutdown() case <-lt.shutdownCtx.Done(): close(sigs) @@ -269,7 +269,7 @@ func (lt *LogTransceiver) handleSignals() { func (lt *LogTransceiver) shutdown() { if atomic.AddUint32(<.closing, 1) == 1 { - common.Log.Debugf("Shutting down log transceiver instance for network: %s", *lt.Network.Name) + common.Log.Debugf("shutting down log transceiver instance for network: %s", *lt.Network.Name) lt.cancelF() } } diff --git a/cmd/statsdaemon/stats_daemon.go b/cmd/statsdaemon/stats_daemon.go index 8bf69de8..a5ec082f 100644 --- a/cmd/statsdaemon/stats_daemon.go +++ b/cmd/statsdaemon/stats_daemon.go @@ -126,7 +126,7 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD useBCInfoWebsocket := false if websocketURL == blockchainInfoWebsocketURL { - common.Log.Debugf("Enabling blockchain.info websocket on network stats websocket for configured network: %s", *network.Name) + common.Log.Debugf("enabling blockchain.info websocket on network stats websocket for configured network: %s", *network.Name) useBCInfoWebsocket = true } @@ -139,14 +139,14 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD var wsDialer websocket.Dialer wsConn, _, err := wsDialer.Dial(websocketURL, nil) if err != nil { - common.Log.Errorf("Failed to establish network stats websocket connection to %s; %s", websocketURL, err.Error()) + common.Log.Errorf("failed to establish network stats websocket connection to %s; %s", websocketURL, err.Error()) } else { defer wsConn.Close() payload := map[string]interface{}{ "op": "ping_block", } if err := wsConn.WriteJSON(payload); err != nil { - common.Log.Errorf("Failed to write ping_block message to blockchain.info network stats websocket connection") + common.Log.Errorf("failed to write ping_block message to blockchain.info network stats websocket connection") return err } @@ -154,25 +154,25 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD "op": "blocks_sub", } if err := wsConn.WriteJSON(payload); err != nil { - common.Log.Errorf("Failed to write block subcription message to blockchain.info network stats websocket connection") + common.Log.Errorf("failed to write block subcription message to blockchain.info network stats websocket connection") } else { - common.Log.Debugf("Subscribed to block headers from blockchain.info network stats websocket: %s", websocketURL) + common.Log.Debugf("subscribed to block headers from blockchain.info network stats websocket: %s", websocketURL) for { _, message, err := wsConn.ReadMessage() if err != nil { - common.Log.Errorf("Failed to receive message on network stats websocket; %s", err) + common.Log.Errorf("failed to receive message on network stats websocket; %s", err) break } else { - common.Log.Debugf("Received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) + common.Log.Tracef("received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) response := map[string]interface{}{} err := json.Unmarshal(message, &response) if err != nil { - common.Log.Warningf("Failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) + common.Log.Warningf("failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) } else { if op, opok := response["op"].(string); opok && op == "block" { if header, headerOk := response["x"].(map[string]interface{}); headerOk { - common.Log.Debugf("Received block header on blockchain.info network stats websocket subscription: %s", websocketURL) + common.Log.Tracef("received block header on blockchain.info network stats websocket subscription: %s", websocketURL) ch <- &provide.NetworkStatus{ Meta: map[string]interface{}{ blockchainInfoWebsocketURL: true, @@ -209,13 +209,13 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD client, err = rpcclient.New(cfg, &rpcclient.NotificationHandlers{ OnClientConnected: func() { - common.Log.Debugf("Bitcoin websocket client connected on configured network stats websocket for network: %s", *network.Name) + common.Log.Debugf("bitcoin websocket client connected on configured network stats websocket for network: %s", *network.Name) // Register for block connect and disconnect notifications. if err := client.NotifyBlocks(); err != nil { } else { if err != nil { - common.Log.Errorf("Failed to establish network stats websocket subscription to %s for network: %s; %s", websocketURL, *network.Name, err.Error()) + common.Log.Errorf("failed to establish network stats websocket subscription to %s for network: %s; %s", websocketURL, *network.Name, err.Error()) client.Disconnect() return } @@ -223,8 +223,8 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD }, OnFilteredBlockConnected: func(height int32, header *wire.BlockHeader, txns []*btcutil.Tx) { - common.Log.Debugf("Received block header on network stats websocket for network: %s; height: %d", *network.Name, height) - common.Log.Debugf("Block connected: %v (%d) %v", header.BlockHash(), height, header.Timestamp) + common.Log.Debugf("received block header on network stats websocket for network: %s; height: %d", *network.Name, height) + common.Log.Debugf("block connected: %v (%d) %v", header.BlockHash(), height, header.Timestamp) ch <- &provide.NetworkStatus{ Meta: map[string]interface{}{ @@ -234,21 +234,21 @@ func BcoinNetworkStatsDataSourceFactory(network *network.Network) *NetworkStatsD }, OnFilteredBlockDisconnected: func(height int32, header *wire.BlockHeader) { - common.Log.Debugf("Received block disconnected header on network stats websocket for network: %s; height: %d", *network.Name, height) - common.Log.Debugf("Block disconnected: %v (%d) %v", header.BlockHash(), height, header.Timestamp) + common.Log.Debugf("received block disconnected header on network stats websocket for network: %s; height: %d", *network.Name, height) + common.Log.Debugf("block disconnected: %v (%d) %v", header.BlockHash(), height, header.Timestamp) }, OnUnknownNotification: func(method string, params []json.RawMessage) { - common.Log.Warningf("Unknown notification received on bitcoin network stats websocket; method: %s; %s", method, params) + common.Log.Warningf("unknown notification received on bitcoin network stats websocket; method: %s; %s", method, params) }, }) if err != nil { - common.Log.Errorf("Failed to establish network stats websocket connection to %s for network: %s; %s", websocketURL, *network.Name, err.Error()) + common.Log.Errorf("failed to establish network stats websocket connection to %s for network: %s; %s", websocketURL, *network.Name, err.Error()) return err } - common.Log.Debugf("Subscribed to network stats websocket: %s", websocketURL) + common.Log.Debugf("subscribed to network stats websocket: %s", websocketURL) client.WaitForShutdown() } @@ -276,7 +276,7 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta var wsDialer websocket.Dialer wsConn, _, err := wsDialer.Dial(websocketURL, nil) if err != nil { - common.Log.Errorf("Failed to establish network stats websocket connection to %s; %s", websocketURL, err.Error()) + common.Log.Errorf("failed to establish network stats websocket connection to %s; %s", websocketURL, err.Error()) } else { defer wsConn.Close() id, _ := uuid.NewV4() @@ -287,21 +287,21 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta "jsonrpc": "2.0", } if err := wsConn.WriteJSON(payload); err != nil { - common.Log.Errorf("Failed to write subscribe message to network stats websocket connection") + common.Log.Errorf("failed to write subscribe message to network stats websocket connection") } else { - common.Log.Debugf("Subscribed to network stats websocket: %s", websocketURL) + common.Log.Debugf("subscribed to network stats websocket: %s", websocketURL) for { _, message, err := wsConn.ReadMessage() if err != nil { - common.Log.Errorf("Failed to receive message on network stats websocket; %s", err) + common.Log.Errorf("failed to receive message on network stats websocket; %s", err) break } else { - common.Log.Debugf("Received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) + common.Log.Tracef("received %d-byte message on network stats websocket for network: %s", len(message), *network.Name) response := &provide.EthereumWebsocketSubscriptionResponse{} err := json.Unmarshal(message, response) if err != nil { - common.Log.Warningf("Failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) + common.Log.Warningf("failed to unmarshal message received on network stats websocket: %s; %s", message, err.Error()) } else { if result, ok := response.Params["result"].(map[string]interface{}); ok { if _, mixHashOk := result["mixHash"]; !mixHashOk { @@ -314,7 +314,7 @@ func EthereumNetworkStatsDataSourceFactory(network *network.Network) *NetworkSta header := &types.Header{} err := json.Unmarshal(resultJSON, header) if err != nil { - common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) + common.Log.Warningf("failed to stringify result JSON in otherwise valid message received on network stats websocket: %s; %s", response, err.Error()) } else if header != nil && header.Number != nil { ch <- &provide.NetworkStatus{ Meta: map[string]interface{}{ @@ -408,31 +408,31 @@ func BaseledgerNetworkStatsDataSourceFactory(network *network.Network) *NetworkS // Consume the websocket stream; attempts to fallback to JSON-RPC if websocket stream fails or is not available for the network func (sd *StatsDaemon) consume() []error { errs := make([]error, 0) - sd.log.Debugf("Attempting to consume configured stats daemon data source; attempt #%v", sd.attempt) + sd.log.Debugf("attempting to consume configured stats daemon data source; attempt #%v", sd.attempt) var err error if sd.dataSource != nil { err = sd.dataSource.Stream(sd.queue) } else { - err = errors.New("Configured stats daemon does not have a configured data source") + err = errors.New("configured stats daemon does not have a configured data source") } if err != nil { errs = append(errs, err) switch err.(type) { case jsonRpcNotSupported: - sd.log.Warningf("Configured stats daemon data source does not support JSON-RPC; attempting to upgrade to websocket stream for network id: %s", sd.dataSource.Network.ID) + sd.log.Warningf("configured stats daemon data source does not support JSON-RPC; attempting to upgrade to websocket stream for network id: %s", sd.dataSource.Network.ID) err := sd.dataSource.Stream(sd.queue) if err != nil { errs = append(errs, err) - sd.log.Warningf("Configured stats daemon data source returned error while consuming JSON-RPC endpoint: %s; restarting stream...", err.Error()) + sd.log.Warningf("configured stats daemon data source returned error while consuming JSON-RPC endpoint: %s; restarting stream...", err.Error()) } case websocketNotSupported: - sd.log.Warningf("Configured stats daemon data source does not support streaming via websocket; attempting to fallback to JSON-RPC long polling using stats daemon for network id: %s", sd.dataSource.Network.ID) + sd.log.Warningf("configured stats daemon data source does not support streaming via websocket; attempting to fallback to JSON-RPC long polling using stats daemon for network id: %s", sd.dataSource.Network.ID) err := sd.dataSource.Poll(sd.queue) if err != nil { errs = append(errs, err) - sd.log.Warningf("Configured stats daemon data source returned error while consuming JSON-RPC endpoint: %s; restarting stream...", err.Error()) + sd.log.Warningf("configured stats daemon data source returned error while consuming JSON-RPC endpoint: %s; restarting stream...", err.Error()) } } } @@ -442,7 +442,7 @@ func (sd *StatsDaemon) consume() []error { func (sd *StatsDaemon) ingest(response interface{}) { defer func() { if r := recover(); r != nil { - common.Log.Warningf("Recovered from failed stats daemon message ingestion attempt; %s", r) + common.Log.Warningf("recovered from failed stats daemon message ingestion attempt; %s", r) } }() @@ -530,7 +530,7 @@ func (sd *StatsDaemon) ingestBcoin(response interface{}) { sd.stats.Syncing = sd.stats.Block == 0 if sd.stats.Block == 0 { - common.Log.Debugf("Ignoring genesis header") + common.Log.Debugf("ignoring genesis header") return } } @@ -545,10 +545,6 @@ func (sd *StatsDaemon) ingestBcoin(response interface{}) { merkleRoot, _ := header["merkleroot"].(string) - // chainptID := fmt.Sprintf("provide.%s.block", sd.dataSource.Network.ID) - // chainptHash := []byte(merkleRoot) - // providechainpoint.ImmortalizeHashes(chainptID, []*[]byte{&chainptHash}) - if len(sd.recentBlocks) == 0 || sd.recentBlocks[len(sd.recentBlocks)-1].(map[string]interface{})["merkleroot"].(string) != merkleRoot { sd.recentBlocks = append(sd.recentBlocks, header) sd.recentBlockTimestamps = append(sd.recentBlockTimestamps, lastBlockAt) @@ -580,27 +576,21 @@ func (sd *StatsDaemon) ingestBcoin(response interface{}) { sd.stats.Meta["last_block_hash"] = merkleRoot } } else if medianTime, medianTimeOk := chainInfo["mediantime"].(float64); medianTimeOk { - // This is pretty naive but gives us an avg. time before we have >= 3 recent blocks; can take some time after statsdaemon starts monitoring a PoW network... + // This is pretty naive but gives us an avg. time before we have >= 3 recent blocks; + // can take some time after statsdaemon starts monitoring a PoW network... sd.stats.Meta["average_blocktime"] = (float64(time.Now().Unix()) - medianTime) / (11.0 / 2.0) } } else { - common.Log.Warningf("Failed to parse last_block_header from *provide.NetworkStats meta; dropping message...") + common.Log.Warningf("failed to parse last_block_header from *provide.NetworkStats meta; dropping message...") } } else { - common.Log.Warningf("Received malformed *provide.NetworkStats message; dropping message...") + common.Log.Warningf("received malformed *provide.NetworkStats message; dropping message...") } } sd.publish() } -func (sd *StatsDaemon) ingestLcoin(response interface{}) { - switch response.(type) { - default: - common.Log.Warningf("Lcoin ingest functionality not yet implemented in stats daemon") - } -} - func (sd *StatsDaemon) ingestEthereum(response interface{}) { switch response.(type) { case *provide.NetworkStatus: @@ -618,16 +608,16 @@ func (sd *StatsDaemon) ingestEthereum(response interface{}) { hdr := &types.Header{} err := json.Unmarshal(headerJSON, hdr) if err != nil { - common.Log.Warningf("Failed to stringify result JSON in otherwise valid message received via JSON-RPC: %s; %s", response, err.Error()) + common.Log.Warningf("failed to stringify result JSON in otherwise valid message received via JSON-RPC: %s; %s", response, err.Error()) } else if hdr != nil && hdr.Number != nil { sd.ingest(hdr) } } } else { - common.Log.Warningf("Failed to parse last_block_header from *provide.NetworkStats meta; dropping message...") + common.Log.Warningf("failed to parse last_block_header from *provide.NetworkStats meta; dropping message...") } } else { - common.Log.Warningf("Received malformed *provide.NetworkStats message; dropping message...") + common.Log.Warningf("received malformed *provide.NetworkStats message; dropping message...") } case *types.Header: header := response.(*types.Header) @@ -636,7 +626,7 @@ func (sd *StatsDaemon) ingestEthereum(response interface{}) { sd.stats.Syncing = sd.stats.Block == 0 if sd.stats.Block == 0 { - common.Log.Debugf("Ignoring genesis header") + common.Log.Debugf("ignoring genesis header") return } @@ -647,10 +637,6 @@ func (sd *StatsDaemon) ingestEthereum(response interface{}) { blockHash := header.Hash().String() - // chainptID := fmt.Sprintf("provide.%s.block", sd.dataSource.Network.ID) - // chainptHash := []byte(blockHash) - // providechainpoint.ImmortalizeHashes(chainptID, []*[]byte{&chainptHash}) - if len(sd.recentBlocks) == 0 || sd.recentBlocks[len(sd.recentBlocks)-1].(*types.Header).Hash().String() != blockHash { sd.recentBlocks = append(sd.recentBlocks, header) sd.recentBlockTimestamps = append(sd.recentBlockTimestamps, lastBlockAt) @@ -705,7 +691,7 @@ func (sd *StatsDaemon) loop() error { sd.ingest(msg) case <-sd.shutdownCtx.Done(): - sd.log.Debugf("Closing stats daemon on shutdown") + sd.log.Debugf("closing stats daemon on shutdown") return nil } } @@ -726,15 +712,17 @@ func (sd *StatsDaemon) publish() error { // EvictNetworkStatsDaemon evicts a single, previously-initialized stats daemon instance { func EvictNetworkStatsDaemon(network *network.Network) error { + currentNetworkStatsMutex.Lock() + defer currentNetworkStatsMutex.Unlock() + if daemon, ok := currentNetworkStats[network.ID.String()]; ok { - common.Log.Debugf("Evicting stats daemon instance for network: %s; id: %s", *network.Name, network.ID) + common.Log.Debugf("evicting stats daemon instance for network: %s; id: %s", *network.Name, network.ID) daemon.shutdown() - currentNetworkStatsMutex.Lock() delete(currentNetworkStats, network.ID.String()) - currentNetworkStatsMutex.Unlock() return nil } - return fmt.Errorf("Unable to evict stats daemon instance for network: %s; id; %s", *network.Name, network.ID) + + return fmt.Errorf("unable to evict stats daemon instance for network: %s; id; %s", *network.Name, network.ID) } // RequireNetworkStatsDaemon ensures a single stats daemon instance is running for @@ -743,18 +731,16 @@ func EvictNetworkStatsDaemon(network *network.Network) error { func RequireNetworkStatsDaemon(network *network.Network) *StatsDaemon { var daemon *StatsDaemon if daemon, ok := currentNetworkStats[network.ID.String()]; ok { - common.Log.Debugf("Cached stats daemon instance found for network: %s; id: %s", *network.Name, network.ID) + common.Log.Debugf("cached stats daemon instance found for network: %s; id: %s", *network.Name, network.ID) return daemon } - currentNetworkStatsMutex.Lock() - common.Log.Infof("Initializing new stats daemon instance for network: %s; id: %s", *network.Name, network.ID) + common.Log.Infof("initializing new stats daemon instance for network: %s; id: %s", *network.Name, network.ID) daemon = NewNetworkStatsDaemon(common.Log, network) if daemon != nil { currentNetworkStats[network.ID.String()] = daemon go daemon.run() } - currentNetworkStatsMutex.Unlock() return daemon } @@ -762,6 +748,9 @@ func RequireNetworkStatsDaemon(network *network.Network) *StatsDaemon { // NewNetworkStatsDaemon initializes a new network stats daemon instance using // NetworkStatsDataSourceFactory to construct the daemon's its data source func NewNetworkStatsDaemon(lg *logger.Logger, network *network.Network) *StatsDaemon { + currentNetworkStatsMutex.Lock() + defer currentNetworkStatsMutex.Unlock() + sd := new(StatsDaemon) sd.attempt = 0 sd.log = lg.Clone() @@ -775,7 +764,7 @@ func NewNetworkStatsDaemon(lg *logger.Logger, network *network.Network) *StatsDa } else if network.IsBaseledgerNetwork() { sd.dataSource = BaseledgerNetworkStatsDataSourceFactory(network) } - //sd.handleSignals() + // sd.handleSignals() if sd.dataSource == nil { return nil @@ -785,7 +774,7 @@ func NewNetworkStatsDaemon(lg *logger.Logger, network *network.Network) *StatsDa if chainID == nil { chn, err := providecrypto.EVMGetChainID(network.ID.String(), network.RPCURL()) if err != nil { - common.Log.Debugf("Error getting chain id for %s network. Error: %s", network.ID.String(), err.Error()) + common.Log.Warningf("failed to retrieve chain id for %s network. Error: %s", network.ID.String(), err.Error()) return nil } _chainID := hexutil.EncodeBig(chn) @@ -805,10 +794,10 @@ func (sd *StatsDaemon) run() error { go func() { for !sd.shuttingDown() { sd.attempt++ - common.Log.Debugf("Stepping into main runloop of stats daemon instance; attempt #%v", sd.attempt) + common.Log.Debugf("stepping into main runloop of stats daemon instance; attempt #%v", sd.attempt) errs := sd.consume() if len(errs) > 0 { - common.Log.Warningf("Configured stats daemon data source returned %v error(s) while attempting to consume configured data source", len(errs)) + common.Log.Warningf("configured stats daemon data source returned %v error(s) while attempting to consume configured data source", len(errs)) if sd.backoff == 0 { sd.backoff = 100 } else { @@ -826,10 +815,10 @@ func (sd *StatsDaemon) run() error { err := sd.loop() if err == nil { - sd.log.Info("Stats daemon exited cleanly") + sd.log.Info("stats daemon exited cleanly") } else { if !sd.shuttingDown() { - common.Log.Errorf("Forcing shutdown of stats daemon due to error; %s", err) + common.Log.Errorf("forcing shutdown of stats daemon due to error; %s", err) sd.shutdown() } } @@ -838,14 +827,14 @@ func (sd *StatsDaemon) run() error { } func (sd *StatsDaemon) handleSignals() { - common.Log.Debug("Installing SIGINT and SIGTERM signal handlers") + common.Log.Debug("installing SIGINT and SIGTERM signal handlers") sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { select { case sig := <-sigs: - common.Log.Infof("Received signal: %s", sig) + common.Log.Infof("received signal: %s", sig) sd.shutdown() case <-sd.shutdownCtx.Done(): close(sigs) @@ -855,7 +844,7 @@ func (sd *StatsDaemon) handleSignals() { func (sd *StatsDaemon) shutdown() { if atomic.AddUint32(&sd.closing, 1) == 1 { - common.Log.Debugf("Shutting down stats daemon instance for network: %s", *sd.dataSource.Network.Name) + common.Log.Debugf("shutting down stats daemon instance for network: %s", *sd.dataSource.Network.Name) sd.cancelF() } } diff --git a/network/consumer.go b/network/consumer.go index ec2ade4c..08f65d2b 100644 --- a/network/consumer.go +++ b/network/consumer.go @@ -16,8 +16,8 @@ import ( ) const natsBlockFinalizedSubject = "nchain.block.finalized" -const natsBlockFinalizedSubjectMaxInFlight = 2048 -const natsBlockFinalizedInvocationTimeout = time.Second * 30 +const natsBlockFinalizedSubjectMaxInFlight = 4096 +const natsBlockFinalizedInvocationTimeout = time.Second * 15 const natsBlockFinalizedTimeout = int64(time.Minute * 1) const natsResolveNodePeerURLSubject = "nchain.node.peer.resolve" @@ -131,18 +131,18 @@ func consumeBlockFinalizedMsg(msg *stan.Msg) { } }() - common.Log.Debugf("Consuming NATS block finalized message: %s", msg) + common.Log.Debugf("consuming NATS block finalized message: %s", msg) var err error blockFinalizedMsg := &natsBlockFinalizedMsg{} err = json.Unmarshal(msg.Data, &blockFinalizedMsg) if err != nil { - common.Log.Warningf("Failed to unmarshal block finalized message; %s", err.Error()) + common.Log.Warningf("failed to unmarshal block finalized message; %s", err.Error()) return } if blockFinalizedMsg.NetworkID == nil { - err = fmt.Errorf("Parsed NATS block finalized message did not contain network id: %s", msg) + err = fmt.Errorf("parsed NATS block finalized message did not contain network id: %s", msg) } if err == nil { @@ -152,7 +152,7 @@ func consumeBlockFinalizedMsg(msg *stan.Msg) { db.Where("id = ?", blockFinalizedMsg.NetworkID).Find(&network) if network == nil || network.ID == uuid.Nil { - err = fmt.Errorf("Failed to retrieve network by id: %s", *blockFinalizedMsg.NetworkID) + err = fmt.Errorf("failed to retrieve network by id: %s", *blockFinalizedMsg.NetworkID) } if err == nil { @@ -193,16 +193,16 @@ func consumeBlockFinalizedMsg(msg *stan.Msg) { } } } else { - err = fmt.Errorf("Failed to decode EVM block header; %s", err.Error()) + err = fmt.Errorf("failed to decode EVM block header; %s", err.Error()) } } else { - common.Log.Warningf("Received unhandled finalized block header; network id: %s", *blockFinalizedMsg.NetworkID) + common.Log.Warningf("received unhandled finalized block header; network id: %s", *blockFinalizedMsg.NetworkID) } } } if err != nil { - common.Log.Warningf("Failed to handle block finalized message; %s", err.Error()) + common.Log.Warningf("failed to handle block finalized message; %s", err.Error()) natsutil.AttemptNack(msg, natsBlockFinalizedTimeout) } else { msg.Ack() @@ -216,12 +216,12 @@ func consumeResolveNodePeerURLMsg(msg *stan.Msg) { } }() - common.Log.Debugf("Consuming NATS resolve node peer url message: %s", msg) + common.Log.Debugf("consuming NATS resolve node peer url message: %s", msg) var params map[string]interface{} err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to umarshal resolve node peer url message; %s", err.Error()) + common.Log.Warningf("failed to umarshal resolve node peer url message; %s", err.Error()) natsutil.Nack(msg) return } @@ -229,7 +229,7 @@ func consumeResolveNodePeerURLMsg(msg *stan.Msg) { nodeID, nodeIDOk := params["node_id"].(string) if !nodeIDOk { - common.Log.Warningf("Failed to resolve peer url for node; no node id provided") + common.Log.Warningf("failed to resolve peer url for node; no node id provided") natsutil.Nack(msg) return } @@ -239,14 +239,14 @@ func consumeResolveNodePeerURLMsg(msg *stan.Msg) { node := &Node{} db.Where("id = ?", nodeID).Find(&node) if node == nil || node.ID == uuid.Nil { - common.Log.Warningf("Failed to resolve node; no node resolved for id: %s", nodeID) + common.Log.Warningf("failed to resolve node; no node resolved for id: %s", nodeID) natsutil.Nack(msg) return } err = node.resolvePeerURL(db) if err != nil { - common.Log.Debugf("Attempt to resolve node peer url did not succeed; %s", err.Error()) + common.Log.Debugf("attempt to resolve node peer url did not succeed; %s", err.Error()) natsutil.AttemptNack(msg, natsResolveNodePeerURLTimeout) return } @@ -267,12 +267,12 @@ func consumeAddNodePeerMsg(msg *stan.Msg) { } }() - common.Log.Debugf("Consuming NATS add peer message: %s", msg) + common.Log.Debugf("consuming NATS add peer message: %s", msg) var params map[string]interface{} err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to umarshal add peer message; %s", err.Error()) + common.Log.Warningf("failed to umarshal add peer message; %s", err.Error()) natsutil.Nack(msg) return } @@ -281,13 +281,13 @@ func consumeAddNodePeerMsg(msg *stan.Msg) { peerURL, peerURLOk := params["peer_url"].(string) if !nodeIDOk { - common.Log.Warningf("Failed to add network peer; no node id provided") + common.Log.Warningf("failed to add network peer; no node id provided") natsutil.Nack(msg) return } if !peerURLOk { - common.Log.Warningf("Failed to add network peer; no peer url provided") + common.Log.Warningf("failed to add network peer; no peer url provided") natsutil.Nack(msg) return } @@ -297,7 +297,7 @@ func consumeAddNodePeerMsg(msg *stan.Msg) { node := &Node{} db.Where("id = ?", nodeID).Find(&node) if node == nil || node.ID == uuid.Nil { - common.Log.Warningf("Failed to resolve node; no node resolved for id: %s", nodeID) + common.Log.Warningf("failed to resolve node; no node resolved for id: %s", nodeID) natsutil.Nack(msg) return } @@ -319,12 +319,12 @@ func consumeRemoveNodePeerMsg(msg *stan.Msg) { } }() - common.Log.Debugf("Consuming NATS remove peer message: %s", msg) + common.Log.Debugf("consuming NATS remove peer message: %s", msg) var params map[string]interface{} err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to umarshal remove peer message; %s", err.Error()) + common.Log.Warningf("failed to umarshal remove peer message; %s", err.Error()) natsutil.Nack(msg) return } @@ -333,13 +333,13 @@ func consumeRemoveNodePeerMsg(msg *stan.Msg) { peerURL, peerURLOk := params["peer_url"].(string) if !nodeIDOk { - common.Log.Warningf("Failed to remove network peer; no node id provided") + common.Log.Warningf("failed to remove network peer; no node id provided") natsutil.Nack(msg) return } if !peerURLOk { - common.Log.Warningf("Failed to remove network peer; no peer url provided") + common.Log.Warningf("failed to remove network peer; no peer url provided") natsutil.Nack(msg) return } @@ -349,14 +349,14 @@ func consumeRemoveNodePeerMsg(msg *stan.Msg) { node := &Node{} db.Where("id = ?", nodeID).Find(&node) if node == nil || node.ID == uuid.Nil { - common.Log.Warningf("Failed to resolve node; no node resolved for id: %s", nodeID) + common.Log.Warningf("failed to resolve node; no node resolved for id: %s", nodeID) natsutil.Nack(msg) return } err = node.removePeer(peerURL) if err != nil { - common.Log.Debugf("Attempt to remove network peer failed; %s", err.Error()) + common.Log.Debugf("attempt to remove network peer failed; %s", err.Error()) natsutil.AttemptNack(msg, natsRemoveNodePeerTimeout) return } diff --git a/tx/consumer.go b/tx/consumer.go index 0d977c2d..b07aadb8 100644 --- a/tx/consumer.go +++ b/tx/consumer.go @@ -14,7 +14,7 @@ import ( stan "github.com/nats-io/stan.go" "github.com/provideplatform/nchain/common" "github.com/provideplatform/nchain/contract" - api "github.com/provideplatform/provide-go/api" + "github.com/provideplatform/provide-go/api" provide "github.com/provideplatform/provide-go/api/nchain" ) @@ -156,7 +156,7 @@ func consumeTxCreateMsg(msg *stan.Msg) { // processTxTransferMsg processes ETH value transfer NATS messages func processTxTransferMsg(msg *stan.Msg) { - common.Log.Debugf("Consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) + common.Log.Debugf("consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) db := dbconf.DatabaseConnection() @@ -164,7 +164,7 @@ func processTxTransferMsg(msg *stan.Msg) { // TODO alter this to support transfers tx, _, err := processNATSTxMsg(msg, db, false) if err != nil { - common.Log.Debugf("Error processing NATS %v message. Error: %s", msg.Subject, err.Error()) + common.Log.Debugf("error processing NATS %v message. error: %s", msg.Subject, err.Error()) natsutil.Nack(msg) return } @@ -172,7 +172,7 @@ func processTxTransferMsg(msg *stan.Msg) { common.Log.Debugf("ACK: Acking previously processed NATS msg seq: %v", msg.Sequence) err := msg.Ack() if err != nil { - common.Log.Debugf("ACK: Error acking previously processed NATS msg seq: %v. Error: %s", msg.Sequence, err.Error()) + common.Log.Debugf("ACK: error acking previously processed NATS msg seq: %v. error: %s", msg.Sequence, err.Error()) natsutil.AttemptNack(msg, txCreateMsgTimeout) } return @@ -185,12 +185,12 @@ func processTxTransferMsg(msg *stan.Msg) { common.Log.Debugf("ConsumeTxExecutionMsg, executed tx with ref: %s", *tx.Ref) } else { - errmsg := fmt.Sprintf("Failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) + errmsg := fmt.Sprintf("failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) for _, err := range tx.Errors { errmsg = fmt.Sprintf("%s\n\t%s", errmsg, *err.Message) } - common.Log.Debugf("Tx ref %s failed. Error: %s, Attempting nacking", *tx.Ref, errmsg) + common.Log.Debugf("Tx ref %s failed. error: %s, Attempting nacking", *tx.Ref, errmsg) natsutil.AttemptNack(msg, txMsgTimeout) } @@ -203,14 +203,14 @@ func processTxTransferMsg(msg *stan.Msg) { // it joins an ordered processing queue func processTxCreateMsg(msg *stan.Msg) { - common.Log.Debugf("Consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) + common.Log.Debugf("consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) db := dbconf.DatabaseConnection() // process msg data tx, contract, err := processNATSTxMsg(msg, db, true) if err != nil { - common.Log.Debugf("Error processing NATS %v message. Error: %s", msg.Subject, err.Error()) + common.Log.Debugf("error processing NATS %v message. error: %s", msg.Subject, err.Error()) natsutil.Nack(msg) return } @@ -218,8 +218,8 @@ func processTxCreateMsg(msg *stan.Msg) { common.Log.Debugf("ACK: Acking previously processed NATS msg seq: %v", msg.Sequence) err := msg.Ack() if err != nil { - common.Log.Debugf("ACK: Error acking previously processed NATS msg seq: %v. Error: %s", msg.Sequence, err.Error()) - //common.Log.Debugf("ACK: Error acking tx ref %s. Error: %s", *tx.Ref, err.Error()) + common.Log.Debugf("ACK: error acking previously processed NATS msg seq: %v. error: %s", msg.Sequence, err.Error()) + //common.Log.Debugf("ACK: error acking tx ref %s. error: %s", *tx.Ref, err.Error()) natsutil.AttemptNack(msg, txCreateMsgTimeout) } return @@ -238,12 +238,12 @@ func processTxCreateMsg(msg *stan.Msg) { } else { - errmsg := fmt.Sprintf("Failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) + errmsg := fmt.Sprintf("failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) for _, err := range tx.Errors { errmsg = fmt.Sprintf("%s\n\t%s", errmsg, *err.Message) } - common.Log.Debugf("Tx ref %s failed. Error: %s, Attempting nacking", *tx.Ref, errmsg) + common.Log.Debugf("Tx ref %s failed. error: %s, Attempting nacking", *tx.Ref, errmsg) natsutil.AttemptNack(msg, txCreateMsgTimeout) } @@ -257,8 +257,8 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to unmarshal tx creation message; %s", err.Error()) - err := fmt.Errorf("error unmarshaling tx creation message from NATS %v message. Error: %s", msg.Subject, err.Error()) + common.Log.Warningf("failed to unmarshal tx creation message; %s", err.Error()) + err := fmt.Errorf("error unmarshaling tx creation message from NATS %v message. error: %s", msg.Subject, err.Error()) return nil, nil, err } @@ -287,8 +287,8 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio // no reference provided with the contract, so we'll make one reference, err = uuid.NewV4() if err != nil { - common.Log.Warningf("Failed to create unique tx ref. Error: %s", err.Error()) - err = fmt.Errorf("error creating unique ref for tx. Error: %s", err.Error()) + common.Log.Warningf("failed to create unique tx ref. error: %s", err.Error()) + err = fmt.Errorf("error creating unique ref for tx. error: %s", err.Error()) return nil, nil, err } } @@ -302,32 +302,32 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio } if !contractIDOk { - common.Log.Warningf("Failed to unmarshal contract_id during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal contract_id during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling contract_id from NATS %v message", msg.Subject) return nil, nil, err } if !dataOk { - common.Log.Warningf("Failed to unmarshal data during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal data during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling data from NATS %v message", msg.Subject) return nil, nil, err } if !accountIDStrOk && !walletIDStrOk { - common.Log.Warningf("Failed to unmarshal account_id or wallet_id during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal account_id or wallet_id during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling both accountID and walletID from NATS %v message", msg.Subject) return nil, nil, err } if !valueOk { - common.Log.Warningf("Failed to unmarshal value during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal value during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling value from NATS %v message", msg.Subject) return nil, nil, err } if !paramsOk { - common.Log.Warningf("Failed to unmarshal params during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal params during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling params from NATS %v message", msg.Subject) return nil, nil, err } if !publishedAtOk { - common.Log.Warningf("Failed to unmarshal published_at during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal published_at during NATS %v message handling", msg.Subject) err := fmt.Errorf("error unmarshaling published_at from NATS %v message", msg.Subject) return nil, nil, err } @@ -349,7 +349,7 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio } if accountID == nil && walletID == nil { - common.Log.Warningf("Failed to unmarshal account_id or wallet_id during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal account_id or wallet_id during NATS %v message handling", msg.Subject) err := fmt.Errorf("error converting accountID and walletID to uuid from NATS %v message", msg.Subject) return nil, nil, err } @@ -374,7 +374,7 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio publishedAtTime, err := time.Parse(time.RFC3339, publishedAt) if err != nil { - common.Log.Warningf("Failed to parse published_at as RFC3339 timestamp during NATS %v message handling; %s", msg.Subject, err.Error()) + common.Log.Warningf("failed to parse published_at as RFC3339 timestamp during NATS %v message handling; %s", msg.Subject, err.Error()) err := fmt.Errorf("error parsing published_at time from NATS %v message", msg.Subject) return nil, nil, err } @@ -382,7 +382,7 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio // convert value to float64 (shouldn't this be uint64?) if it's present valueFloat, valueFloatOk := value.(float64) if !valueFloatOk { - common.Log.Warningf("Failed to unmarshal value during NATS %v message handling", msg.Subject) + common.Log.Warningf("failed to unmarshal value during NATS %v message handling", msg.Subject) err := fmt.Errorf("error converting value to float64 from NATS %v message", msg.Subject) return nil, nil, err } @@ -450,7 +450,7 @@ func processNATSTxMsg(msg *stan.Msg, db *gorm.DB, newContract bool) (*Transactio // replaces this tx with the db tx (if it exists and matches) replacementTx, err := tx.replaceWithDatabaseTxIfExists(db, ref) if err != nil { - common.Log.Debugf("Error retrieving tx ref %s from database", *tx.Ref) + common.Log.Debugf("error retrieving tx ref %s from database", *tx.Ref) return nil, nil, err } @@ -552,48 +552,44 @@ func (tx *Transaction) replaceWithDatabaseTxIfExists(db *gorm.DB, ref string) (* func consumeTxExecutionMsg(msg *stan.Msg) { start := time.Now() - common.Log.Debugf("TIMINGNANO: about to process nats sequence %v", msg.Sequence) + common.Log.Tracef("processing tx message; sequence: %v", msg.Sequence) processTxExecutionMsg(msg) - elapsedTime := time.Since(start) - common.Log.Debugf("TIMINGNANO: processed nats sequence %v in %s", msg.Sequence, elapsedTime) + common.Log.Tracef("processed tx message; sequence: %v in %s", msg.Sequence, time.Since(start)) } func processTxExecutionMsg(msg *stan.Msg) { - - common.Log.Debugf("Consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) - + common.Log.Debugf("consuming %d-byte NATS tx message on subject: %s", msg.Size(), msg.Subject) db := dbconf.DatabaseConnection() // process msg data tx, _, err := processNATSTxMsg(msg, db, false) if err != nil { - common.Log.Debugf("Error processing NATS %v message. Error: %s", msg.Subject, err.Error()) + common.Log.Debugf("error processing NATS %v message. error: %s", msg.Subject, err.Error()) natsutil.Nack(msg) return } if tx == nil { - common.Log.Debugf("ACK: Acking previously processed NATS msg seq: %v", msg.Sequence) + common.Log.Debugf("ACK: acking previously processed NATS msg seq: %v", msg.Sequence) err := msg.Ack() if err != nil { - common.Log.Debugf("ACK: Error acking previously processed NATS msg seq: %v. Error: %s", msg.Sequence, err.Error()) + common.Log.Debugf("ACK: error acking previously processed NATS msg seq: %v. error: %s", msg.Sequence, err.Error()) natsutil.AttemptNack(msg, txCreateMsgTimeout) } return } - common.Log.Debugf("ConsumeTxExecutionMsg, about to execute tx with ref: %s", *tx.Ref) + common.Log.Tracef("attempting to execute tx with ref: %s", *tx.Ref) // check if we have this tx ref in the database if tx.Create(db) { common.Log.Debugf("ConsumeTxExecutionMsg, executed tx with ref: %s", *tx.Ref) } else { - - errmsg := fmt.Sprintf("Failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) + errmsg := fmt.Sprintf("failed to execute transaction; tx ref %s failed with %d error(s)", *tx.Ref, len(tx.Errors)) for _, err := range tx.Errors { errmsg = fmt.Sprintf("%s\n\t%s", errmsg, *err.Message) } - common.Log.Debugf("Tx ref %s failed. Error: %s, Attempting nacking", *tx.Ref, errmsg) + common.Log.Debugf("Tx ref %s failed. error: %s, Attempting nacking", *tx.Ref, errmsg) natsutil.AttemptNack(msg, txMsgTimeout) } @@ -603,13 +599,13 @@ func processTxExecutionMsg(msg *stan.Msg) { // TODO: consider batching this using a buffered channel for high-volume networks func consumeTxFinalizeMsg(msg *stan.Msg) { - common.Log.Tracef("Consuming NATS tx finalize message: %s", msg) + common.Log.Tracef("consuming NATS tx finalize message: %s", msg) var params map[string]interface{} nack := func(msg *stan.Msg, errmsg string, dropPacket bool) { if dropPacket { - common.Log.Tracef("Dropping tx packet (seq: %d) on the floor; %s", msg.Sequence, errmsg) + common.Log.Tracef("dropping tx packet on the floor; seq: %d; %s", msg.Sequence, errmsg) msg.Ack() return } @@ -618,7 +614,7 @@ func consumeTxFinalizeMsg(msg *stan.Msg) { err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - nack(msg, fmt.Sprintf("Failed to unmarshal tx finalize message; %s", err.Error()), true) + nack(msg, fmt.Sprintf("failed to unmarshal tx finalize message; %s", err.Error()), true) return } @@ -628,32 +624,32 @@ func consumeTxFinalizeMsg(msg *stan.Msg) { hash, hashOk := params["hash"].(string) if !blockOk { - nack(msg, "Failed to finalize tx; no block provided", true) + nack(msg, "failed to finalize tx; no block provided", true) return } if !blockTimestampStrOk { - nack(msg, "Failed to finalize tx; no block timestamp provided", true) + nack(msg, "failed to finalize tx; no block timestamp provided", true) return } if !finalizedAtStrOk { - nack(msg, "Failed to finalize tx; no finalized at timestamp provided", true) + nack(msg, "failed to finalize tx; no finalized at timestamp provided", true) return } if !hashOk { - nack(msg, "Failed to finalize tx; no hash provided", true) + nack(msg, "failed to finalize tx; no hash provided", true) return } blockTimestamp, err := time.Parse(time.RFC3339, blockTimestampStr) if err != nil { - nack(msg, fmt.Sprintf("Failed to unmarshal block_timestamp during NATS %v message handling; %s", msg.Subject, err.Error()), true) + nack(msg, fmt.Sprintf("failed to unmarshal block_timestamp during NATS %v message handling; %s", msg.Subject, err.Error()), true) return } finalizedAt, err := time.Parse(time.RFC3339, finalizedAtStr) if err != nil { - nack(msg, fmt.Sprintf("Failed to unmarshal finalized_at during NATS %v message handling; %s", msg.Subject, err.Error()), true) + nack(msg, fmt.Sprintf("failed to unmarshal finalized_at during NATS %v message handling; %s", msg.Subject, err.Error()), true) return } @@ -663,7 +659,7 @@ func consumeTxFinalizeMsg(msg *stan.Msg) { db.Where("hash = ? AND status IN (?, ?, ?, ?)", hash, "ready", "broadcast", "pending", "failed").Find(&tx) if tx == nil || tx.ID == uuid.Nil { // TODO: this is integration point to upsert Wallet & Transaction... need to think thru performance implications & implementation details - nack(msg, fmt.Sprintf("Failed to mark block and finalized_at timestamp on tx: %s; tx not found for given hash", hash), true) + nack(msg, fmt.Sprintf("failed to mark block and finalized_at timestamp on tx: %s; tx not found for given hash", hash), true) return } @@ -694,10 +690,10 @@ func consumeTxFinalizeMsg(msg *stan.Msg) { Message: common.StringOrNil(err.Error()), }) } - common.Log.Debugf("IDEMPOTENT: error updating tx ref %s. Error: %s", *tx.Ref, err.Error()) + common.Log.Warningf("error updating tx; ref: %s; error: %s", *tx.Ref, err.Error()) } if len(tx.Errors) > 0 { - nack(msg, fmt.Sprintf("Failed to set block and finalized_at timestamp on tx: %s; error: %s", hash, *tx.Errors[0].Message), false) + nack(msg, fmt.Sprintf("failed to set block and finalized_at timestamp on tx: %s; error: %s", hash, *tx.Errors[0].Message), false) return } @@ -705,7 +701,6 @@ func consumeTxFinalizeMsg(msg *stan.Msg) { } func processTxReceipt(msg *stan.Msg, tx *Transaction, key *string, db *gorm.DB) { - signer, err := tx.signerFactory(db) if err != nil { desc := "failed to resolve tx signing account or HD wallet" @@ -718,18 +713,17 @@ func processTxReceipt(msg *stan.Msg, tx *Transaction, key *string, db *gorm.DB) address, err := signer.Address() if err != nil { // let's try processing this one again - common.Log.Debugf(fmt.Sprintf("Failed to get account address from signer for tx ref %s. Error: %s", *tx.Ref, err.Error())) + common.Log.Debugf(fmt.Sprintf("failed to get account address from signer for tx ref %s. error: %s", *tx.Ref, err.Error())) natsutil.AttemptNack(msg, txReceiptMsgTimeout) return } err = tx.fetchReceipt(db, signer.Network, *address) if err != nil { - // TODO got a panic here on *tx.hash (removed temporarily) - common.Log.Debugf(fmt.Sprintf("Failed to fetch tx receipt for tx ref %s. Error: %s", *tx.Ref, err.Error())) + common.Log.Debugf(fmt.Sprintf("failed to fetch tx receipt for tx ref %s. error: %s", *tx.Ref, err.Error())) natsutil.AttemptNack(msg, txReceiptMsgTimeout) } else { - common.Log.Debugf("Fetched tx receipt for tx ref %s, hash: %s", *tx.Ref, *tx.Hash) + common.Log.Debugf("fetched tx receipt for tx ref %s, hash: %s", *tx.Ref, *tx.Hash) blockNumber := tx.Response.Receipt.(*provide.TxReceipt).BlockNumber // if we have a block number in the receipt, and the tx has no block @@ -753,7 +747,7 @@ func processTxReceipt(msg *stan.Msg, tx *Transaction, key *string, db *gorm.DB) tx.NetworkLatency = &networkLatency } - common.Log.Debugf("*** tx hash %s finalized in block %v at %s", *tx.Hash, blockNumber, receiptFinalized.Format("Mon, 02 Jan 2006 15:04:05 MST")) + common.Log.Tracef("tx %s finalized in block %v at %s", *tx.Hash, blockNumber, receiptFinalized.Format("Mon, 02 Jan 2006 15:04:05 MST")) } tx.updateStatus(db, "success", nil) msg.Ack() @@ -761,8 +755,6 @@ func processTxReceipt(msg *stan.Msg, tx *Transaction, key *string, db *gorm.DB) } func consumeTxReceiptMsg(msg *stan.Msg) { - var key *string - defer func() { if r := recover(); r != nil { common.Log.Warningf("recovered from failed tx receipt message; %s", r) @@ -770,9 +762,10 @@ func consumeTxReceiptMsg(msg *stan.Msg) { } }() - common.Log.Debugf("consuming NATS tx receipt message: %s", msg) + common.Log.Tracef("consuming NATS tx receipt message: %s", msg) var params map[string]interface{} + var key *string err := json.Unmarshal(msg.Data, ¶ms) if err != nil { @@ -788,7 +781,7 @@ func consumeTxReceiptMsg(msg *stan.Msg) { return } - common.Log.Debugf("Consuming tx receipt for tx id: %s", transactionID) + common.Log.Debugf("consuming tx receipt for tx id: %s", transactionID) db := dbconf.DatabaseConnection() tx := &Transaction{} @@ -799,9 +792,6 @@ func consumeTxReceiptMsg(msg *stan.Msg) { return } - common.Log.Debugf("Starting processTxReceipt for tx ref: %s", *tx.Ref) - // TODO occasionally throws panic on startup - // caused by vault not being available, so the signer.address is not populated (:803-ish) - // process the receipts asynchronously + common.Log.Tracef("attempting to asynchronously process tx receipt; ref: %s", *tx.Ref) go processTxReceipt(msg, tx, key, db) } diff --git a/tx/shuttle_consumer.go b/tx/shuttle_consumer.go index ab6c876a..b0ec9f10 100644 --- a/tx/shuttle_consumer.go +++ b/tx/shuttle_consumer.go @@ -68,13 +68,13 @@ func createNatsShuttleContractDeployedSubject(wg *sync.WaitGroup) { } func consumeShuttleCircuitDeployedMsg(msg *stan.Msg) { - common.Log.Debugf("Consuming NATS shuttle circuit deployed message: %s", msg) + common.Log.Debugf("consuming NATS shuttle circuit deployed message: %s", msg) var params map[string]interface{} err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to umarshal shuttle circuit deployed message; %s", err.Error()) + common.Log.Warningf("failed to umarshal shuttle circuit deployed message; %s", err.Error()) natsutil.Nack(msg) return } @@ -84,13 +84,13 @@ func consumeShuttleCircuitDeployedMsg(msg *stan.Msg) { } func consumeShuttleContractDeployedMsg(msg *stan.Msg) { - common.Log.Debugf("Consuming NATS shuttle contract deployed message: %s", msg) + common.Log.Debugf("consuming NATS shuttle contract deployed message: %s", msg) var params map[string]interface{} err := json.Unmarshal(msg.Data, ¶ms) if err != nil { - common.Log.Warningf("Failed to umarshal shuttle contract deployed message; %s", err.Error()) + common.Log.Warningf("failed to umarshal shuttle contract deployed message; %s", err.Error()) natsutil.Nack(msg) return } @@ -103,31 +103,31 @@ func consumeShuttleContractDeployedMsg(msg *stan.Msg) { contractType, _ := params["type"].(string) if !addressOk { - common.Log.Warning("Failed to handle shuttle.contract.deployed message; contract address required") + common.Log.Warning("failed to handle shuttle.contract.deployed message; contract address required") natsutil.Nack(msg) return } if !byOk { - common.Log.Warning("Failed to handle shuttle.contract.deployed message; by address required") + common.Log.Warning("failed to handle shuttle.contract.deployed message; by address required") natsutil.Nack(msg) return } if !networkIDOk { - common.Log.Warning("Failed to handle shuttle.contract.deployed message; contract network_id required") + common.Log.Warning("failed to handle shuttle.contract.deployed message; contract network_id required") natsutil.Nack(msg) return } if !nameOk { - common.Log.Warning("Failed to handle shuttle.contract.deployed message; contract name required") + common.Log.Warning("failed to handle shuttle.contract.deployed message; contract name required") natsutil.Nack(msg) return } if !txHashOk { - common.Log.Warning("Failed to handle shuttle.contract.deployed message; tx hash required") + common.Log.Warning("failed to handle shuttle.contract.deployed message; tx hash required") natsutil.Nack(msg) return } @@ -137,14 +137,14 @@ func consumeShuttleContractDeployedMsg(msg *stan.Msg) { db.Where("network_id = ? AND address = ?", networkID, byAddr).Find(&cntrct) if cntrct == nil || cntrct.ID == uuid.Nil { - common.Log.Warningf("Failed to handle shuttle.contract.deployed message; contract not resolved for address: %s", byAddr) + common.Log.Warningf("failed to handle shuttle.contract.deployed message; contract not resolved for address: %s", byAddr) natsutil.AttemptNack(msg, natsShuttleContractDeployedTimeout) return } network, err := cntrct.GetNetwork() if err != nil { - common.Log.Warningf("Failed to handle shuttle.contract.deployed message; network not resolved for contract with address: %s; %s", byAddr, err.Error()) + common.Log.Warningf("failed to handle shuttle.contract.deployed message; network not resolved for contract with address: %s; %s", byAddr, err.Error()) natsutil.AttemptNack(msg, natsShuttleContractDeployedTimeout) return } @@ -156,21 +156,21 @@ func consumeShuttleContractDeployedMsg(msg *stan.Msg) { p2pAPI, err := network.P2PAPIClient() if err != nil { - common.Log.Warningf("Failed to handle shuttle.contract.deployed message; network P2P API client not resolved for contract with address: %s; %s", byAddr, err.Error()) + common.Log.Warningf("failed to handle shuttle.contract.deployed message; network P2P API client not resolved for contract with address: %s; %s", byAddr, err.Error()) natsutil.AttemptNack(msg, natsShuttleContractDeployedTimeout) return } receipt, err := p2pAPI.FetchTxReceipt(*cntrct.Address, txHash) if err != nil { - common.Log.Warningf("Failed to handle shuttle.contract.deployed message; failed to fetch tx receipt for contract with address: %s; %s", byAddr, err.Error()) + common.Log.Warningf("failed to handle shuttle.contract.deployed message; failed to fetch tx receipt for contract with address: %s; %s", byAddr, err.Error()) natsutil.AttemptNack(msg, natsShuttleContractDeployedTimeout) return } dependency := cntrct.ResolveCompiledDependencyArtifact(name) if dependency == nil { - common.Log.Warningf("Failed to handle shuttle.contract.deployed message; contract at address %s unable to resolved dependency: %s", byAddr, name) + common.Log.Warningf("failed to handle shuttle.contract.deployed message; contract at address %s unable to resolved dependency: %s", byAddr, name) natsutil.AttemptNack(msg, natsShuttleContractDeployedTimeout) return }