Add the following: (#501)

1. chainregistry package used for interacting with the Git repo Cosmos chain registry
2. Functionality for pulling the repo and parsing asset lists
3. Use the new package in the following places:
    * Client initialization setup to get and cache asset mappings
    * Denom unit conversion method to check for asset list entry and prefer that over database denoms

.gitignore

@@ -14,3 +14,4 @@ vendor/
 tax.txt
 tax-db-pwd.txt
 log.txt
+registry

chainregistry/chainregistry.go

@@ -0,0 +1,52 @@
+package chainregistry
+
+type AssetList struct {
+	ChainName string  `json:"chain_name"`
+	Assets    []Asset `json:"assets"`
+}
+
+type Asset struct {
+	Description string           `json:"description"`
+	Base        string           `json:"base"`
+	Symbol      string           `json:"symbol"`
+	DenomUnits  []AssetDenomUnit `json:"denom_units"`
+	ChainName   string           `json:"chain_name,omitempty"`
+}
+
+type AssetDenomUnit struct {
+	Denom    string `json:"denom"`
+	Exponent uint   `json:"exponent"`
+}
+
+var CachedAssetMap = map[string]Asset{}
+
+func CacheAssetMap(newVal map[string]Asset) {
+	CachedAssetMap = newVal
+}
+
+func GetCachedAssetEntry(denomBase string) (Asset, bool) {
+	asset, ok := CachedAssetMap[denomBase]
+	return asset, ok
+}
+
+func GetBaseDenomUnitForAsset(asset Asset) AssetDenomUnit {
+	lowestDenomUnit := AssetDenomUnit{Exponent: 0}
+	for _, denomUnit := range asset.DenomUnits {
+		if denomUnit.Exponent <= lowestDenomUnit.Exponent {
+			lowestDenomUnit = denomUnit
+		}
+	}
+
+	return lowestDenomUnit
+}
+
+func GetHighestDenomUnitForAsset(asset Asset) AssetDenomUnit {
+	highestDenomUnit := AssetDenomUnit{Exponent: 0}
+	for _, denomUnit := range asset.DenomUnits {
+		if denomUnit.Exponent >= highestDenomUnit.Exponent {
+			highestDenomUnit = denomUnit
+		}
+	}
+
+	return highestDenomUnit
+}

chainregistry/requests.go

@@ -0,0 +1,106 @@
+package chainregistry
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/DefiantLabs/cosmos-tax-cli/config"
+	"github.com/go-git/go-git/v5"
+)
+
+const (
+	ChainRegistryGitRepo = "https://github.com/cosmos/chain-registry.git"
+)
+
+func UpdateChainRegistryOnDisk(chainRegistryLocation string) error {
+	_, err := os.Stat(chainRegistryLocation)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if os.IsNotExist(err) {
+		err = os.Mkdir(chainRegistryLocation, 0o777)
+		if err != nil {
+			return err
+		}
+	}
+
+	// git clone repo
+	_, err = git.PlainClone(chainRegistryLocation, false, &git.CloneOptions{
+		URL: ChainRegistryGitRepo,
+	})
+
+	// Check if already cloned
+	if err != nil && !errors.Is(err, git.ErrRepositoryAlreadyExists) {
+		return err
+	} else if errors.Is(err, git.ErrRepositoryAlreadyExists) {
+		// Pull if already cloned
+		r, err := git.PlainOpen(chainRegistryLocation)
+		if err != nil {
+			return err
+		}
+
+		w, err := r.Worktree()
+		if err != nil {
+			return err
+		}
+
+		err = w.Pull(&git.PullOptions{})
+		// Ignore up-to-date error
+		if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func GetAssetMapOnDisk(chainRegistryLocation string, chainRegBlacklist map[string]bool) (map[string]Asset, error) {
+	chainRegEntries, err := os.ReadDir(chainRegistryLocation)
+	if err != nil {
+		return nil, err
+	}
+	assetMap := make(map[string]Asset)
+	for _, entry := range chainRegEntries {
+		if entry.IsDir() {
+			inBlacklist := chainRegBlacklist[entry.Name()]
+			if !inBlacklist {
+				path := fmt.Sprintf("%s/%s/assetlist.json", chainRegistryLocation, entry.Name())
+
+				// check if file exists
+				_, err := os.Stat(path)
+				if err != nil && os.IsNotExist(err) {
+					config.Log.Warnf("Chain registry asset list for %s does not exist. Skipping...", entry.Name())
+					continue
+				} else if err != nil {
+					return nil, err
+				}
+
+				// load asset list
+				jsonFile, err := os.Open(path)
+				if err != nil {
+					return nil, err
+				}
+
+				currAssets := &AssetList{}
+				err = json.NewDecoder(jsonFile).Decode(currAssets)
+
+				if err != nil {
+					return nil, err
+				}
+
+				for _, asset := range currAssets.Assets {
+					asset.ChainName = currAssets.ChainName
+					if prevEntry, ok := assetMap[asset.Base]; ok {
+						config.Log.Warnf("Duplicate asset found for %s in %s. Overwriting entry for %s", asset.Base, currAssets.ChainName, prevEntry.ChainName)
+					}
+					assetMap[asset.Base] = asset
+				}
+			}
+		}
+	}
+
+	return assetMap, nil
+}

client/client.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/DefiantLabs/cosmos-tax-cli/chainregistry"
 	"github.com/DefiantLabs/cosmos-tax-cli/client/docs"
 	"github.com/DefiantLabs/cosmos-tax-cli/config"
 	"github.com/DefiantLabs/cosmos-tax-cli/csv"
@@ -24,6 +25,29 @@ var (
 	ClientCfg *config.ClientConfig
 )
 
+var chainRegBlacklist = map[string]bool{
+	"_IBC":        true,
+	"_memo_keys":  true,
+	"_non-cosmos": true,
+	"_template":   true,
+	".github":     true,
+	".git":        true,
+	"testnets":    true,
+	"thorchain":   true,
+	"xion":        true,
+}
+
+func loadChainRegistryAssetLists(cfg *config.ClientConfig) (map[string]chainregistry.Asset, error) {
+	err := chainregistry.UpdateChainRegistryOnDisk(cfg.ChainRegistryLocation)
+	if err != nil {
+		return nil, err
+	}
+
+	assetMap, err := chainregistry.GetAssetMapOnDisk(cfg.ChainRegistryLocation, chainRegBlacklist)
+
+	return assetMap, err
+}
+
 func setup() (*gorm.DB, *config.ClientConfig, int, string, error) {
 	argConfig, flagSet, svcPort, err := config.ParseClientArgs(os.Stderr, os.Args[1:])
 	if err != nil {
@@ -98,6 +122,13 @@ func main() {
 	DB = db
 	ClientCfg = cfg
 
+	assetMap, err := loadChainRegistryAssetLists(ClientCfg)
+	if err != nil {
+		config.Log.Fatalf("Error loading chain registry. Err: %v", err)
+	}
+
+	chainregistry.CacheAssetMap(assetMap)
+
 	// Have to keep this here so that import of docs subfolder (which contains proper init()) stays
 	docs.SwaggerInfo.Title = "Cosmos Tax CLI"
 

config/client_config.go

@@ -12,10 +12,11 @@ import (
 )
 
 type ClientConfig struct {
-	ConfigFileLocation string
-	Database           Database
-	Client             client
-	Log                log
+	ConfigFileLocation    string
+	ChainRegistryLocation string
+	Database              Database
+	Client                client
+	Log                   log
 }
 
 type client struct {
@@ -28,6 +29,7 @@ func ParseClientArgs(w io.Writer, args []string) (ClientConfig, *flag.FlagSet, i
 
 	fs.SetOutput(w)
 	fs.StringVar(&c.ConfigFileLocation, "config", "", "The file to load for configuration variables")
+	fs.StringVar(&c.ChainRegistryLocation, "chain-registry", "./registry", "The folder containing the chain registry files")
 
 	// Database
 	fs.StringVar(&c.Database.Host, "db.host", "", "The PostgreSQL hostname for the indexer db")

db/denoms.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/DefiantLabs/cosmos-tax-cli/chainregistry"
 	"gorm.io/gorm"
 )
 
@@ -120,30 +121,57 @@ func ConvertUnits(amount *big.Int, denom Denom) (*big.Float, string, error) {
 		return new(big.Float).Quo(convertedAmount, new(big.Float).SetFloat64(power)), denom.Base, nil
 	}
 
-	// Try denom unit first
-	// We were originally just using GetDenomUnitForDenom, but since CachedDenoms is an array, it would sometimes
-	// return the non-Base denom unit (exponent != 0), which would break the power conversion process below i.e.
-	// it would sometimes do highestDenomUnit.Exponent = 6, denomUnit.Exponent = 6 -> pow = 0
-	denomUnit, err := GetBaseDenomUnitForDenom(denom)
-	if err != nil {
-		fmt.Println("Error getting denom unit for denom", denom)
-		return nil, "", fmt.Errorf("error getting denom unit for denom %+v", denom)
+	// Try chainregistry asset lists first
+	// We are experimenting with a full pull-down of the asset list entries in the chain registry to see if
+	// they provide good coverage for parsing items into symbols.
+	base := denom.Base
+	if strings.HasPrefix(base, "transfer/") {
+		splitString := strings.Split(denom.Base, "/")
+		base = splitString[len(splitString)-1]
 	}
 
-	highestDenomUnit, err := GetHighestDenomUnit(denomUnit, CachedDenomUnits)
-	if err != nil {
-		fmt.Println("Error getting highest denom unit for denom", denom)
-		return nil, "", fmt.Errorf("error getting highest denom unit for denom %+v", denom)
-	}
+	assetEntry, ok := chainregistry.GetCachedAssetEntry(base)
+
+	var symbol string
+	var highestExponent uint
+	var baseExponent uint
+	var highestExponentName string
+	if ok {
+		baseExponent = chainregistry.GetBaseDenomUnitForAsset(assetEntry).Exponent
+		highestDenomUnit := chainregistry.GetHighestDenomUnitForAsset(assetEntry)
+		highestExponent = highestDenomUnit.Exponent
+		highestExponentName = highestDenomUnit.Denom
+		symbol = assetEntry.Symbol
+	} else {
+
+		// Try denom unit second
+		// We were originally just using GetDenomUnitForDenom, but since CachedDenoms is an array, it would sometimes
+		// return the non-Base denom unit (exponent != 0), which would break the power conversion process below i.e.
+		// it would sometimes do highestDenomUnit.Exponent = 6, denomUnit.Exponent = 6 -> pow = 0
+		denomUnit, err := GetBaseDenomUnitForDenom(denom)
+		if err != nil {
+			fmt.Println("Error getting denom unit for denom", denom)
+			return nil, "", fmt.Errorf("error getting denom unit for denom %+v", denom)
+		}
 
-	symbol := denomUnit.Denom.Symbol
+		highestDenomUnit, err := GetHighestDenomUnit(denomUnit, CachedDenomUnits)
+		if err != nil {
+			fmt.Println("Error getting highest denom unit for denom", denom)
+			return nil, "", fmt.Errorf("error getting highest denom unit for denom %+v", denom)
+		}
 
+		symbol = denomUnit.Denom.Symbol
+		highestExponent = highestDenomUnit.Exponent
+		baseExponent = denomUnit.Exponent
+		highestExponentName = highestDenomUnit.Name
+	}
 	// We were converting the units to big.Int, which would cause a Token to appear 0 if the conversion resulted in an amount < 1
-	power := math.Pow(10, float64(highestDenomUnit.Exponent-denomUnit.Exponent))
+	power := math.Pow(10, float64(highestExponent-baseExponent))
 	dividedAmount := new(big.Float).Quo(convertedAmount, new(big.Float).SetFloat64(power))
 	if symbol == "UNKNOWN" || symbol == "" {
-		symbol = highestDenomUnit.Name
+		symbol = highestExponentName
 	}
+
 	return dividedAmount, symbol, nil
 }