feat: fix http lcd errors (#28)

* fix: fix HTTP LCD errors

* chore: fix error

* feat: add custom unmarshaller for SingleBlockResponse

* chore: refactor models

* feat: testing block unmarshal

* chore: fixed linting

* feat: testing validators response unmarshal

* chore: fixed linting

pkg/tendermint/http.go

@@ -6,6 +6,7 @@ import (
 	configPkg "main/pkg/config"
 	"main/pkg/constants"
 	"main/pkg/metrics"
+	"main/pkg/types/responses"
 	"main/pkg/utils"
 	"net/http"
 	"net/url"
@@ -46,15 +47,15 @@ func (rpc *RPC) GetConsumerOrProviderHosts() []string {
 	return rpc.config.RPCEndpoints
 }
 
-func (rpc *RPC) GetBlock(height int64) (*types.SingleBlockResponse, error) {
+func (rpc *RPC) GetBlock(height int64) (*responses.SingleBlockResponse, error) {
 	queryURL := "/block"
 	if height != 0 {
 		queryURL = fmt.Sprintf("/block?height=%d", height)
 	}
 
-	var response types.SingleBlockResponse
+	var response responses.SingleBlockResponse
 	if err := rpc.Get(queryURL, constants.QueryTypeBlock, &response, rpc.config.RPCEndpoints, func(v interface{}) error {
-		response, ok := v.(*types.SingleBlockResponse)
+		response, ok := v.(*responses.SingleBlockResponse)
 		if !ok {
 			return fmt.Errorf("error converting block")
 		}
@@ -99,9 +100,9 @@ func (rpc *RPC) AbciQuery(
 		queryURL += fmt.Sprintf("&height=%d", height)
 	}
 
-	var response types.AbciQueryResponse
+	var response responses.AbciQueryResponse
 	if err := rpc.Get(queryURL, constants.QueryType("abci_"+string(queryType)), &response, hosts, func(v interface{}) error {
-		response, ok := v.(*types.AbciQueryResponse)
+		response, ok := v.(*responses.AbciQueryResponse)
 		if !ok {
 			return fmt.Errorf("error converting ABCI response")
 		}
@@ -244,9 +245,9 @@ func (rpc *RPC) GetActiveSetAtBlock(height int64) (map[string]bool, error) {
 			page,
 		)
 
-		var response types.ValidatorsResponse
+		var response responses.ValidatorsResponse
 		if err := rpc.Get(queryURL, constants.QueryTypeHistoricalValidators, &response, rpc.config.RPCEndpoints, func(v interface{}) error {
-			response, ok := v.(*types.ValidatorsResponse)
+			response, ok := v.(*responses.ValidatorsResponse)
 			if !ok {
 				return fmt.Errorf("error converting validators")
 			}
@@ -287,10 +288,15 @@ func (rpc *RPC) Get(
 ) error {
 	errors := make([]error, len(hosts))
 
-	indexes := utils.MakeShuffledArray(len(hosts))
+	indexesShuffled := utils.MakeShuffledArray(len(hosts))
+	hostsShuffled := make([]string, len(hosts))
 
-	for _, index := range indexes {
-		lcd := hosts[index]
+	for index, indexShuffled := range indexesShuffled {
+		hostsShuffled[index] = hosts[indexShuffled]
+	}
+
+	for index := range indexesShuffled {
+		lcd := hostsShuffled[index]
 
 		fullURL := lcd + url
 		rpc.logger.Trace().Str("url", fullURL).Msg("Trying making request to LCD")
@@ -310,7 +316,7 @@ func (rpc *RPC) Get(
 
 		if predicateErr := predicate(target); predicateErr != nil {
 			rpc.logger.Warn().Str("url", fullURL).Err(predicateErr).Msg("LCD precondition failed")
-			errors[index] = fmt.Errorf("precondition failed")
+			errors[index] = fmt.Errorf("precondition failed: %s", predicateErr)
 			continue
 		}
 
@@ -322,7 +328,7 @@ func (rpc *RPC) Get(
 	var sb strings.Builder
 
 	sb.WriteString("All LCD requests failed:\n")
-	for index, nodeURL := range hosts {
+	for index, nodeURL := range hostsShuffled {
 		sb.WriteString(fmt.Sprintf("#%d: %s -> %s\n", index+1, nodeURL, errors[index]))
 	}
 
@@ -334,7 +340,7 @@ func (rpc *RPC) GetFull(
 	queryType constants.QueryType,
 	target interface{},
 ) error {
-	client := &http.Client{Timeout: 10 * 1000000000}
+	client := &http.Client{Timeout: 10 * time.Second}
 	start := time.Now()
 
 	fullURL := host + url

pkg/tendermint/http_manager.go

@@ -3,7 +3,7 @@ package tendermint
 import (
 	configPkg "main/pkg/config"
 	"main/pkg/metrics"
-	"main/pkg/types"
+	"main/pkg/types/responses"
 	"sync"
 
 	providerTypes "github.com/cosmos/interchain-security/x/ccv/provider/types"
@@ -23,7 +23,7 @@ func NewRPCManager(config *configPkg.ChainConfig, logger zerolog.Logger, metrics
 	return &RPCManager{rpc: rpc}
 }
 
-func (manager *RPCManager) GetBlock(height int64) (*types.SingleBlockResponse, error) {
+func (manager *RPCManager) GetBlock(height int64) (*responses.SingleBlockResponse, error) {
 	return manager.rpc.GetBlock(height)
 }
 
@@ -55,11 +55,11 @@ func (manager *RPCManager) GetActiveSetAtBlock(height int64) (map[string]bool, e
 }
 
 func (manager *RPCManager) GetBlocksAndValidatorsAtHeights(heights []int64) (
-	map[int64]*types.SingleBlockResponse,
+	map[int64]*responses.SingleBlockResponse,
 	map[int64]map[string]bool,
 	[]error,
 ) {
-	blocksMap := make(map[int64]*types.SingleBlockResponse)
+	blocksMap := make(map[int64]*responses.SingleBlockResponse)
 	activeSetsMap := make(map[int64]map[string]bool)
 	errors := make([]error, 0)
 

pkg/tendermint/websocket.go

@@ -6,6 +6,7 @@ import (
 	configPkg "main/pkg/config"
 	"main/pkg/constants"
 	"main/pkg/metrics"
+	"main/pkg/types/responses"
 	"reflect"
 	"strings"
 	"time"
@@ -188,7 +189,7 @@ func (t *WebsocketClient) ProcessEvent(event rpcTypes.RPCResponse) {
 	t.metricsManager.LogWSEvent(t.config.Name, t.url)
 	t.lastEventTime = time.Now()
 
-	var resultEvent types.EventResult
+	var resultEvent responses.EventResult
 	if err := json.Unmarshal(event.Result, &resultEvent); err != nil {
 		t.logger.Error().Err(err).Msg("Failed to parse event")
 		t.Channel <- &types.WSError{Error: err}
@@ -211,7 +212,7 @@ func (t *WebsocketClient) ProcessEvent(event rpcTypes.RPCResponse) {
 		return
 	}
 
-	var blockData types.SingleBlockResult
+	var blockData responses.SingleBlockResult
 	if err := json.Unmarshal(blockDataStr, &blockData); err != nil {
 		t.logger.Error().Err(err).Msg("Failed to unmarshall event")
 		return

pkg/types/responses/abci.go

@@ -0,0 +1,15 @@
+package responses
+
+type AbciQueryResponse struct {
+	Result AbciQueryResult `json:"result"`
+}
+
+type AbciQueryResult struct {
+	Response AbciResponse `json:"response"`
+}
+
+type AbciResponse struct {
+	Code  int    `json:"code"`
+	Log   string `json:"log"`
+	Value []byte `json:"value"`
+}

pkg/types/responses.go → pkg/types/responses/block.go

@@ -1,13 +1,56 @@
-package types
+package responses
 
 import (
+	"encoding/json"
+	"main/pkg/types"
 	"strconv"
 	"time"
 )
 
+func (s *SingleBlockResponse) UnmarshalJSON(data []byte) error {
+	var v map[string]interface{}
+	if err := json.Unmarshal(data, &v); err != nil {
+		return err
+	}
+
+	if result, ok := v["result"]; !ok {
+		s.Result = nil
+	} else {
+		rawBytes, err := json.Marshal(result)
+		if err != nil {
+			return err
+		}
+
+		var resultParsed SingleBlockResult
+		if err := json.Unmarshal(rawBytes, &resultParsed); err != nil {
+			return err
+		}
+
+		s.Result = &resultParsed
+	}
+
+	if responseError, ok := v["error"]; !ok {
+		s.Error = nil
+	} else {
+		rawBytes, err := json.Marshal(responseError)
+		if err != nil {
+			return err
+		}
+
+		var errorParsed ResponseError
+		if err := json.Unmarshal(rawBytes, &errorParsed); err != nil {
+			return err
+		}
+
+		s.Error = &errorParsed
+	}
+
+	return nil
+}
+
 type SingleBlockResponse struct {
-	Result SingleBlockResult `json:"result"`
-	Error  *ResponseError    `json:"error"`
+	Result *SingleBlockResult `json:"result"`
+	Error  *ResponseError     `json:"error"`
 }
 
 type SingleBlockResult struct {
@@ -38,7 +81,7 @@ type BlockSignature struct {
 	ValidatorAddress string `json:"validator_address"`
 }
 
-func (b *TendermintBlock) ToBlock() (*Block, error) {
+func (b *TendermintBlock) ToBlock() (*types.Block, error) {
 	height, err := strconv.ParseInt(b.Header.Height, 10, 64)
 	if err != nil {
 		return nil, err
@@ -50,49 +93,10 @@ func (b *TendermintBlock) ToBlock() (*Block, error) {
 		signatures[signature.ValidatorAddress] = int32(signature.BlockIDFlag)
 	}
 
-	return &Block{
+	return &types.Block{
 		Height:     height,
 		Time:       b.Header.Time,
 		Proposer:   b.Header.Proposer,
 		Signatures: signatures,
 	}, nil
 }
-
-type EventResult struct {
-	Query string    `json:"query"`
-	Data  EventData `json:"data"`
-}
-
-type EventData struct {
-	Type  string                 `json:"type"`
-	Value map[string]interface{} `json:"value"`
-}
-
-type AbciQueryResponse struct {
-	Result AbciQueryResult `json:"result"`
-}
-
-type AbciQueryResult struct {
-	Response AbciResponse `json:"response"`
-}
-
-type AbciResponse struct {
-	Code  int    `json:"code"`
-	Log   string `json:"log"`
-	Value []byte `json:"value"`
-}
-
-type ValidatorsResponse struct {
-	Result ValidatorsResult `json:"result"`
-	Error  *ResponseError   `json:"error"`
-}
-
-type ValidatorsResult struct {
-	Validators []HistoricalValidator `json:"validators"`
-	Count      string                `json:"count"`
-	Total      string                `json:"total"`
-}
-
-type HistoricalValidator struct {
-	Address string `json:"address"`
-}

pkg/types/responses/block_test.go

@@ -0,0 +1,63 @@
+package responses
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestToBlockInvalid(t *testing.T) {
+	t.Parallel()
+
+	blockRaw := &TendermintBlock{
+		Header: BlockHeader{Height: "invalid"},
+	}
+
+	block, err := blockRaw.ToBlock()
+	assert.NotNil(t, err, "Error should be presented!")
+	assert.Nil(t, block, "Block should not be presented!")
+}
+
+func TestToBlockValid(t *testing.T) {
+	t.Parallel()
+
+	blockRaw := &TendermintBlock{
+		Header: BlockHeader{Height: "100"},
+		LastCommit: BlockLastCommit{
+			Signatures: []BlockSignature{
+				{ValidatorAddress: "first", BlockIDFlag: 1},
+				{ValidatorAddress: "second", BlockIDFlag: 2},
+			},
+		},
+	}
+
+	block, err := blockRaw.ToBlock()
+	assert.Nil(t, err, "Error should not be presented!")
+	assert.NotNil(t, block, "Block should be presented!")
+	assert.Equalf(t, block.Height, int64(100), "Block height mismatch!")
+	assert.Len(t, block.Signatures, 2, "Block should have 2 signatures!")
+	assert.Equal(t, block.Signatures["first"], int32(1), "Block signature mismatch!")
+	assert.Equal(t, block.Signatures["second"], int32(2), "Block signature mismatch!")
+}
+
+func TestBlockResponseUnmarshalJson(t *testing.T) {
+	t.Parallel()
+
+	successJSON := "{\"jsonrpc\":\"2.0\",\"id\":-1,\"result\":{\"block\":{\"header\":{\"height\":\"12938640\",\"time\":\"2023-09-30T12:31:56.119728652Z\",\"proposer_address\":\"9F478F8D407008B415BA721548A8A2D010254E19\"},\"last_commit\":{\"signatures\":[{\"block_id_flag\":2,\"validator_address\":\"F57E65CB3534A939E1C428241640B9458F6C458D\",\"timestamp\":\"2023-09-30T12:31:56.247075163Z\",\"signature\":\"H/A0W4UnJlDGXpPyOFHu+Yr0nKzECo3HXKdpT6QLt4S7kVptCQiJHdf3dqdVwcEv971HZe7Qt0viiq/toyAlCA==\"}]}}}}"
+	errorJSON := "{\"jsonrpc\":\"2.0\",\"id\":-1,\"error\":{\"code\":-32603,\"message\":\"Internal error\",\"data\":\"height 10158584 is not available, lowest height is 12308055\"}}"
+
+	var blockResponse SingleBlockResponse
+
+	err := json.Unmarshal([]byte(successJSON), &blockResponse)
+
+	assert.Nil(t, err, "Should not error unmarshalling JSON!")
+	assert.Nil(t, blockResponse.Error, "Unmarshall mismatch!")
+	assert.NotNil(t, blockResponse.Result, "Unmarshall mismatch!")
+
+	err2 := json.Unmarshal([]byte(errorJSON), &blockResponse)
+
+	assert.Nil(t, err2, "Should not error unmarshalling JSON!")
+	assert.NotNil(t, blockResponse.Error, "Unmarshall mismatch!")
+	assert.Nil(t, blockResponse.Result, "Unmarshall mismatch!")
+}

pkg/types/responses/event.go

@@ -0,0 +1,11 @@
+package responses
+
+type EventResult struct {
+	Query string    `json:"query"`
+	Data  EventData `json:"data"`
+}
+
+type EventData struct {
+	Type  string                 `json:"type"`
+	Value map[string]interface{} `json:"value"`
+}