Merge pull request #556 from DefiantLabs/pharr117/555-missing-codecs-…

…tx-parse-error patch/#555 missing codecs tx parse error
DefiantLabs · Jun 26, 2024 · 9e1ffc6 · 9e1ffc6
2 parents 169febb + 6e6ab2e
commit 9e1ffc6
Show file tree

Hide file tree

Showing 20 changed files with 64 additions and 53 deletions.
diff --git a/chainregistry/requests.go b/chainregistry/requests.go
@@ -86,7 +86,6 @@ func GetAssetMapOnDisk(chainRegistryLocation string, chainRegBlacklist map[strin
 
 				currAssets := &AssetList{}
 				err = json.NewDecoder(jsonFile).Decode(currAssets)
-
 				if err != nil {
 					return nil, err
 				}

diff --git a/client/client.go b/client/client.go
@@ -102,7 +102,6 @@ func setup() (*gorm.DB, *config.ClientConfig, int, string, error) {
 	dbTypes.CacheIBCDenoms(db)
 
 	err = db.AutoMigrate(&dbTypes.Address{}, &AddressUsageCSV{}, &AddressUsageJSON{})
-
 	if err != nil {
 		config.Log.Fatalf("Error migrating client models. Err: %v", err)
 	}

diff --git a/cmd/block_enqueue.go b/cmd/block_enqueue.go
@@ -75,7 +75,6 @@ func (idxr *Indexer) enqueueBlocksToProcessFromBlockInputFile(blockChan chan int
 	}
 	var blocksToIndex []uint64
 	err = json.Unmarshal(plan, &blocksToIndex)
-
 	if err != nil {
 		errString := err.Error()
 

diff --git a/cmd/index.go b/cmd/index.go
@@ -328,7 +328,7 @@ func processBlock(cl *client.ChainClient, dbConn *gorm.DB, failedBlockHandler fu
 	var err error
 	errTypeURL := false
 
-	txsEventResp, err := rpc.GetTxsByBlockHeight(cl, newBlock.Height)
+	txsEventResp, unpackError, err := rpc.GetTxsByBlockHeight(cl, newBlock.Height)
 	if err != nil {
 		if strings.Contains(err.Error(), "unable to resolve type URL") {
 			errTypeURL = true
@@ -338,6 +338,10 @@ func processBlock(cl *client.ChainClient, dbConn *gorm.DB, failedBlockHandler fu
 		}
 	}
 
+	if unpackError != nil {
+		config.Log.Warn("There was an error unpacking interfaces during RPC querying, check your codecs", unpackError)
+	}
+
 	// There are two reasons this block would be hit
 	// 1) The node might have pruned history resulting in a failed lookup. Recheck to see if the block was supposed to have TX results.
 	// 2) The RPC endpoint (node we queried) doesn't recognize the type URL anymore, for an older type (e.g. on an archive node).
@@ -728,7 +732,6 @@ func (idxr *Indexer) doDBUpdates(wg *sync.WaitGroup, txDataChan chan *dbData, bl
 			}
 
 			err = dbTypes.UpdateEpochIndexingStatus(idxr.db, idxr.dryRun, epochEventData.epochNumber, epochEventData.epochIdentifier, idxr.cfg.Lens.ChainID, idxr.cfg.Lens.ChainName)
-
 			if err != nil {
 				config.Log.Fatal(fmt.Sprintf("Error indexing block events for %s. Could not mark Epoch indexed.", identifierLoggingString), err)
 			}

diff --git a/config/index_config.go b/config/index_config.go
@@ -83,15 +83,13 @@ func (conf *IndexConfig) Validate() error {
 	lensConf := conf.Lens
 
 	lensConf, err = validateLensConf(lensConf)
-
 	if err != nil {
 		return err
 	}
 
 	conf.Lens = lensConf
 
 	err = validateThrottlingConf(conf.Base.throttlingBase)
-
 	if err != nil {
 		return err
 	}

diff --git a/config/lens_helper.go b/config/lens_helper.go
@@ -38,6 +38,6 @@ func GetLensConfig(conf lens, debug bool) *lensClient.ChainClientConfig {
 		Timeout:        "30s",
 		OutputFormat:   "json",
 		SignModeStr:    "direct",
-		Modules:        lensClient.ModuleBasics,
+		Modules:        lensClient.OsmosisModuleBasics,
 	}
 }
diff --git a/config/update_denoms_config.go b/config/update_denoms_config.go
@@ -29,7 +29,6 @@ func (conf *UpdateDenomsConfig) Validate() error {
 	lensConf := conf.Lens
 
 	lensConf, err = validateLensConf(lensConf)
-
 	if err != nil {
 		return err
 	}

diff --git a/config/update_epochs_config.go b/config/update_epochs_config.go
@@ -31,15 +31,13 @@ func (conf *UpdateEpochsConfig) Validate() error {
 	lensConf := conf.Lens
 
 	lensConf, err = validateLensConf(lensConf)
-
 	if err != nil {
 		return err
 	}
 
 	conf.Lens = lensConf
 
 	err = validateThrottlingConf(conf.Base.throttlingBase)
-
 	if err != nil {
 		return err
 	}

diff --git a/core/tx.go b/core/tx.go
@@ -283,7 +283,6 @@ func ProcessRPCBlockByHeightTXs(db *gorm.DB, cl *client.ChainClient, blockResult
 		// We can entirely ignore failed TXs in downstream parsers, because according to the Cosmos specification, a single failed message in a TX fails the whole TX
 		if txResult.Code == 0 {
 			logs, err = types.ParseABCILogs(txResult.Log)
-
 			if err != nil {
 				logs, err = indexerEvents.ParseTxEventsToMessageIndexEvents(len(txFull.Body.Messages), txResult.Events)
 			}

diff --git a/cosmos/modules/ibc/types.go b/cosmos/modules/ibc/types.go
@@ -229,5 +229,5 @@ func (w *WrapperMsgAcknowledgement) String() string {
 		return "MsgAcknowledgement: IBC transfer was not a FungibleTokenTransfer"
 	}
 
-	return fmt.Sprintf("MsgAcknowledgement: IBC transfer of %s%s from %s to %s\n", w.Amount, w.Denom, w.SenderAddress, w.ReceiverAddress)
+	return fmt.Sprintf("MsgAcknowledgement: IBC transfer of %s%s from %s to %s", w.Amount, w.Denom, w.SenderAddress, w.ReceiverAddress)
 }
diff --git a/cosmos/modules/tx/errors.go b/cosmos/modules/tx/errors.go
@@ -13,5 +13,5 @@ type MessageLogFormatError struct {
 }
 
 func (e *MessageLogFormatError) Error() string {
-	return fmt.Sprintf("Type: %s could not handle message log %s\n", e.MessageType, e.Log)
+	return fmt.Sprintf("Type: %s could not handle message log %s", e.MessageType, e.Log)
 }
diff --git a/csv/parsers/accointing/accointing.go b/csv/parsers/accointing/accointing.go
@@ -158,28 +158,28 @@ func HandleFees(address string, events []db.TaxableTransaction, allFees []db.Fee
 	// We need to gather all unique fees, but we are receiving Messages not Txes
 	// Make a map from TX hash to fees array to keep unique
 	txToFeesMap := make(map[uint][]db.Fee)
-	txIdsToTx := make(map[uint]db.Tx)
+	txIDsToTX := make(map[uint]db.Tx)
 	for _, event := range events {
 		txID := event.Message.Tx.ID
 		feeStore := event.Message.Tx.Fees
 		txToFeesMap[txID] = feeStore
-		txIdsToTx[txID] = event.Message.Tx
+		txIDsToTX[txID] = event.Message.Tx
 	}
 
 	// Due to the way we are parsing, we may have fees for TX that we don't have events for
 	for _, fee := range allFees {
 		txID := fee.Tx.ID
 		if _, ok := txToFeesMap[txID]; !ok {
 			txToFeesMap[txID] = []db.Fee{fee}
-			txIdsToTx[txID] = fee.Tx
+			txIDsToTX[txID] = fee.Tx
 		}
 	}
 
 	for id, txFees := range txToFeesMap {
 		for _, fee := range txFees {
 			if fee.PayerAddress.Address == address {
 				newRow := Row{}
-				err = newRow.ParseFee(txIdsToTx[id], fee)
+				err = newRow.ParseFee(txIDsToTX[id], fee)
 				if err != nil {
 					return nil, err
 				}

diff --git a/csv/parsers/koinly/koinly.go b/csv/parsers/koinly/koinly.go
@@ -29,7 +29,7 @@ var unsupportedCoins = []string{
 
 var coinReplacementMap = map[string]string{}
 
-var symbolsToKoinlyIds = map[string]string{}
+var symbolsToKoinlyIDs = map[string]string{}
 
 // Probably not the best place for this, but its only used in Koinly for now
 // May want to consider adding the asset list URL as a config value and passing the AssetList value down to parsers if needed
@@ -42,7 +42,7 @@ func init() {
 
 	for _, asset := range assetList.Assets {
 		// Required format for koinly IDs is ID:<val>
-		symbolsToKoinlyIds[asset.Symbol] = fmt.Sprintf("ID:%s", asset.KoinlyID)
+		symbolsToKoinlyIDs[asset.Symbol] = fmt.Sprintf("ID:%s", asset.KoinlyID)
 	}
 }
 
@@ -124,18 +124,18 @@ func (p *Parser) GetRows(address string, startDate, endDate *time.Time) ([]parse
 	for i, row := range koinlyRows {
 
 		if row.FeeCurrency != "" {
-			if _, ok := symbolsToKoinlyIds[row.FeeCurrency]; ok {
-				koinlyRows[i].FeeCurrency = symbolsToKoinlyIds[row.FeeCurrency]
+			if _, ok := symbolsToKoinlyIDs[row.FeeCurrency]; ok {
+				koinlyRows[i].FeeCurrency = symbolsToKoinlyIDs[row.FeeCurrency]
 			}
 		}
 		if row.ReceivedCurrency != "" {
-			if _, ok := symbolsToKoinlyIds[row.ReceivedCurrency]; ok {
-				koinlyRows[i].ReceivedCurrency = symbolsToKoinlyIds[row.ReceivedCurrency]
+			if _, ok := symbolsToKoinlyIDs[row.ReceivedCurrency]; ok {
+				koinlyRows[i].ReceivedCurrency = symbolsToKoinlyIDs[row.ReceivedCurrency]
 			}
 		}
 		if row.SentCurrency != "" {
-			if _, ok := symbolsToKoinlyIds[row.SentCurrency]; ok {
-				koinlyRows[i].SentCurrency = symbolsToKoinlyIds[row.SentCurrency]
+			if _, ok := symbolsToKoinlyIDs[row.SentCurrency]; ok {
+				koinlyRows[i].SentCurrency = symbolsToKoinlyIDs[row.SentCurrency]
 			}
 		}
 	}
@@ -226,28 +226,28 @@ func HandleFees(address string, events []db.TaxableTransaction, allFees []db.Fee
 	// We need to gather all unique fees, but we are receiving Messages not Txes
 	// Make a map from TX hash to fees array to keep unique
 	txToFeesMap := make(map[uint][]db.Fee)
-	txIdsToTx := make(map[uint]db.Tx)
+	txIDsToTX := make(map[uint]db.Tx)
 	for _, event := range events {
 		txID := event.Message.Tx.ID
 		feeStore := event.Message.Tx.Fees
 		txToFeesMap[txID] = feeStore
-		txIdsToTx[txID] = event.Message.Tx
+		txIDsToTX[txID] = event.Message.Tx
 	}
 
 	// Due to the way we are parsing, we may have fees for TX that we don't have events for
 	for _, fee := range allFees {
 		txID := fee.Tx.ID
 		if _, ok := txToFeesMap[txID]; !ok {
 			txToFeesMap[txID] = []db.Fee{fee}
-			txIdsToTx[txID] = fee.Tx
+			txIDsToTX[txID] = fee.Tx
 		}
 	}
 
 	for id, txFees := range txToFeesMap {
 		for _, fee := range txFees {
 			if fee.PayerAddress.Address == address {
 				newRow := Row{}
-				err = newRow.ParseFee(txIdsToTx[id], fee)
+				err = newRow.ParseFee(txIDsToTX[id], fee)
 				if err != nil {
 					return nil, err
 				}

diff --git a/csv/parsers/taxbit/taxbit.go b/csv/parsers/taxbit/taxbit.go
@@ -155,28 +155,28 @@ func HandleFees(address string, events []db.TaxableTransaction, allFees []db.Fee
 	// We need to gather all unique fees, but we are receiving Messages not Txes
 	// Make a map from TX hash to fees array to keep unique
 	txToFeesMap := make(map[uint][]db.Fee)
-	txIdsToTx := make(map[uint]db.Tx)
+	txIDsToTX := make(map[uint]db.Tx)
 	for _, event := range events {
 		txID := event.Message.Tx.ID
 		feeStore := event.Message.Tx.Fees
 		txToFeesMap[txID] = feeStore
-		txIdsToTx[txID] = event.Message.Tx
+		txIDsToTX[txID] = event.Message.Tx
 	}
 
 	// Due to the way we are parsing, we may have fees for TX that we don't have events for
 	for _, fee := range allFees {
 		txID := fee.Tx.ID
 		if _, ok := txToFeesMap[txID]; !ok {
 			txToFeesMap[txID] = []db.Fee{fee}
-			txIdsToTx[txID] = fee.Tx
+			txIDsToTX[txID] = fee.Tx
 		}
 	}
 
 	for id, txFees := range txToFeesMap {
 		for _, fee := range txFees {
 			if fee.PayerAddress.Address == address {
 				newRow := Row{}
-				err = newRow.ParseFee(txIdsToTx[id], fee)
+				err = newRow.ParseFee(txIDsToTX[id], fee)
 				if err != nil {
 					return nil, err
 				}

diff --git a/go.mod b/go.mod
@@ -6,9 +6,8 @@ toolchain go1.21.11
 
 require (
 	github.com/BurntSushi/toml v1.3.2
-	// FYI, you can do go get github.com/DefiantLabs/lens@1f6f34841280df179c6e098f040bd584ced43a4c
-	// (using the commit hash from github) to pin to a specific commit.
-	github.com/DefiantLabs/lens v0.3.1-0.20240530040544-3ad0f04acd32
+	// Our lens library is pinned to this specific SHA and branch: 7732945f81598e4d6aff47e0d4ca81b0ec26d8b3, https://github.com/DefiantLabs/lens/tree/v0.0.21-dl
+	github.com/DefiantLabs/lens v0.3.1-0.20240626071018-7732945f8159
 	github.com/cosmos/cosmos-sdk v0.47.8
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-co-op/gocron v1.13.0
@@ -93,6 +92,9 @@ require (
 	github.com/cosmos/gogogateway v1.2.0 // indirect
 	github.com/cosmos/gogoproto v1.4.11 // indirect
 	github.com/cosmos/iavl v1.1.2-0.20240405173644-e52f7630d3b7 // indirect
+	github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v7 v7.1.3 // indirect
+	github.com/cosmos/ibc-apps/modules/async-icq/v7 v7.1.1 // indirect
+	github.com/cosmos/ibc-go/modules/light-clients/08-wasm v0.1.1-ibc-go-v7.3-wasmvm-v1.5 // indirect
 	github.com/cosmos/ics23/go v0.10.0 // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.4 // indirect
 	github.com/cosmos/rosetta-sdk-go v0.10.0 // indirect
@@ -161,6 +163,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
 	github.com/huandu/skiplist v1.2.0 // indirect
+	github.com/iancoleman/orderedmap v0.3.0 // indirect
 	github.com/improbable-eng/grpc-web v0.15.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -204,6 +207,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/osmosis-labs/osmosis/osmomath v0.0.13 // indirect
 	github.com/osmosis-labs/osmosis/osmoutils v0.0.13 // indirect
+	github.com/osmosis-labs/osmosis/x/ibc-hooks v0.0.15 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect

diff --git a/go.sum b/go.sum
@@ -558,8 +558,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
 github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/DefiantLabs/lens v0.3.1-0.20240530040544-3ad0f04acd32 h1:wl/7DINIB9ekq3/Hm9Ck0T0y9oA49YdLvAUh8AsTRXE=
-github.com/DefiantLabs/lens v0.3.1-0.20240530040544-3ad0f04acd32/go.mod h1:+PbjDba4Lz2mDEywmIv3DKuWH2G/asWovif+Dph6zd8=
+github.com/DefiantLabs/lens v0.3.1-0.20240626071018-7732945f8159 h1:GB/lSc4/UT7M64scdyh0lrwyRRi3UyCto9RvwJekVKI=
+github.com/DefiantLabs/lens v0.3.1-0.20240626071018-7732945f8159/go.mod h1:nTPf+NQkhPH6NcBY5p0lpdb14WPiZOeMZDDkq1IWl30=
 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
@@ -1799,7 +1799,11 @@ github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
 github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -1887,6 +1891,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
+go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=

diff --git a/osmosis/modules/gamm/exits.go b/osmosis/modules/gamm/exits.go
@@ -195,7 +195,6 @@ func (sf *WrapperMsgExitSwapShareAmountIn) HandleMsg(msgType string, msg sdk.Msg
 		return &txModule.MessageLogFormatError{MessageType: msgType, Log: fmt.Sprintf("%+v", log)}
 	}
 	sf.TokenOut, err = sdk.ParseCoinNormalized(tokenOut)
-
 	if err != nil {
 		return &txModule.MessageLogFormatError{MessageType: msgType, Log: fmt.Sprintf("%+v", log)}
 	}
@@ -357,7 +356,6 @@ func (sf *WrapperMsgExitSwapExternAmountOut) HandleMsg(msgType string, msg sdk.M
 		return &txModule.MessageLogFormatError{MessageType: msgType, Log: fmt.Sprintf("%+v", log)}
 	}
 	sf.TokenOut, err = sdk.ParseCoinNormalized(tokenOut)
-
 	if err != nil {
 		return &txModule.MessageLogFormatError{MessageType: msgType, Log: fmt.Sprintf("%+v", log)}
 	}

diff --git a/rest/requests.go b/rest/requests.go
@@ -65,7 +65,6 @@ func GetTxsByBlockHeight(host string, height uint64) (tx.GetTxByBlockHeightRespo
 	defer resp.Body.Close()
 
 	err = checkResponseErrorCode(requestEndpoint, resp)
-
 	if err != nil {
 		return result, err
 	}
@@ -76,7 +75,6 @@ func GetTxsByBlockHeight(host string, height uint64) (tx.GetTxByBlockHeightRespo
 	}
 
 	err = json.Unmarshal(body, &result)
-
 	if err != nil {
 		return result, err
 	}
@@ -97,7 +95,6 @@ func GetLatestBlock(host string) (tx.GetLatestBlockResponse, error) {
 	defer resp.Body.Close()
 
 	err = checkResponseErrorCode(requestEndpoint, resp)
-
 	if err != nil {
 		return result, err
 	}
@@ -108,7 +105,6 @@ func GetLatestBlock(host string) (tx.GetLatestBlockResponse, error) {
 	}
 
 	err = json.Unmarshal(body, &result)
-
 	if err != nil {
 		return result, err
 	}