Address usage models for client api

client/client.go

@@ -101,6 +101,12 @@ func setup() (*gorm.DB, *config.ClientConfig, int, string, error) {
 	dbTypes.CacheDenoms(db)
 	dbTypes.CacheIBCDenoms(db)
 
+	err = db.AutoMigrate(&dbTypes.Address{}, &AddressUsageCSV{}, &AddressUsageJSON{})
+
+	if err != nil {
+		config.Log.Fatalf("Error migrating client models. Err: %v", err)
+	}
+
 	return db, &cfg, svcPort, cfg.Client.Model, nil
 }
 
@@ -258,15 +264,37 @@ func GetTaxableEventsCSV(c *gin.Context) {
 		return
 	}
 
-	accountRows, headers, err := csv.ParseForAddress(addresses, startDate, endDate, DB, format)
+	// We only want to process and return data on addresses we know are valid (in our DB)
+	validAddresses, validAddressDBObjects, err := GetValidAddresses(addresses)
+	if err != nil {
+		config.Log.Errorf("Error getting valid addresses: %v", addresses)
+		c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck
+		return
+	}
+
+	accountRows, headers, addressRowsCount, err := csv.ParseForAddress(validAddresses, startDate, endDate, DB, format)
 	if err != nil {
 		// the error returned here has already been pushed to the context... I think.
-		config.Log.Errorf("Error getting rows for addresses: %v", addresses)
+		config.Log.Errorf("Error getting valid addresses %v: %s", addresses, err)
 		fmt.Println(err)
 		c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck
 		return
 	}
 
+	for _, address := range validAddressDBObjects {
+		usage := AddressUsageCSV{
+			Address:       address,
+			AddressID:     address.ID,
+			RowsRetrieved: addressRowsCount[address.Address],
+			Timestamp:     time.Now(),
+		}
+
+		err = DB.Create(&usage).Error
+		if err != nil {
+			config.Log.Errorf("Error saving address usage: %v", err)
+		}
+	}
+
 	if len(accountRows) == 0 {
 		c.JSON(404, gin.H{"message": "No transactions for given address"})
 		return
@@ -306,14 +334,36 @@ func GetTaxableEventsJSON(c *gin.Context) {
 		return
 	}
 
-	accountRows, _, err := csv.ParseForAddress(addresses, startDate, endDate, DB, format)
+	// We only want to process and return data on addresses we know are valid (in our DB)
+	validAddresses, validAddressDBObjects, err := GetValidAddresses(addresses)
+	if err != nil {
+		config.Log.Errorf("Error getting valid addresses %v: %s", addresses, err)
+		c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck
+		return
+	}
+
+	accountRows, _, addressRowsCount, err := csv.ParseForAddress(validAddresses, startDate, endDate, DB, format)
 	if err != nil {
 		// the error returned here has already been pushed to the context... I think.
-		config.Log.Errorf("Error getting rows for addresses: %v", addresses)
+		config.Log.Errorf("Error getting rows for addresses: %v", validAddresses)
 		c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck
 		return
 	}
 
+	for _, address := range validAddressDBObjects {
+		usage := AddressUsageJSON{
+			Address:       address,
+			AddressID:     address.ID,
+			RowsRetrieved: addressRowsCount[address.Address],
+			Timestamp:     time.Now(),
+		}
+
+		err = DB.Create(&usage).Error
+		if err != nil {
+			config.Log.Errorf("Error saving address usage: %v", err)
+		}
+	}
+
 	if len(accountRows) == 0 {
 		c.JSON(404, gin.H{"message": "No transactions for given address"})
 		return
@@ -374,3 +424,49 @@ func ParseTaxableEventsBody(c *gin.Context) ([]string, string, *time.Time, *time
 
 	return addresses, format, startDate, endDate, nil
 }
+
+func GetValidAddresses(addresses []string) ([]string, []dbTypes.Address, error) {
+	validAddresses, err := dbTypes.GetAddresses(addresses, DB)
+	if err != nil {
+		config.Log.Errorf("Error getting addresses: %v", addresses)
+		return nil, nil, err
+	}
+
+	var validAddressesStr []string
+	if len(validAddresses) != len(addresses) {
+		for _, address := range addresses {
+			found := false
+			for _, validAddress := range validAddresses {
+				if address == validAddress.Address {
+					found = true
+					break
+				}
+			}
+			if found {
+				validAddressesStr = append(validAddressesStr, address)
+			} else {
+				config.Log.Infof("Invalid address not found in DB, skipping: %v", address)
+			}
+		}
+	} else {
+		validAddressesStr = addresses
+	}
+
+	return validAddressesStr, validAddresses, nil
+}
+
+type AddressUsageCSV struct {
+	ID            uint
+	Address       dbTypes.Address
+	AddressID     uint
+	RowsRetrieved uint
+	Timestamp     time.Time
+}
+
+type AddressUsageJSON struct {
+	ID            uint
+	Address       dbTypes.Address
+	AddressID     uint
+	RowsRetrieved uint
+	Timestamp     time.Time
+}

cmd/query.go

@@ -50,7 +50,7 @@ var queryCmd = &cobra.Command{
 			endDate = &parsedDate
 		}
 
-		csvRows, headers, err := csv.ParseForAddress(queryConfig.Base.Addresses, startDate, endDate, db, queryConfig.Base.Format)
+		csvRows, headers, _, err := csv.ParseForAddress(queryConfig.Base.Addresses, startDate, endDate, db, queryConfig.Base.Format)
 		if err != nil {
 			log.Println(queryConfig.Base.Addresses)
 			config.Log.Fatal("Error calling parser for address", err)

csv/parser.go

@@ -45,63 +45,67 @@ func GetParser(parserKey string) parsers.Parser {
 	return nil
 }
 
-func ParseForAddress(addresses []string, startDate, endDate *time.Time, pgSQL *gorm.DB, parserKey string) ([]parsers.CsvRow, []string, error) {
+func ParseForAddress(addresses []string, startDate, endDate *time.Time, pgSQL *gorm.DB, parserKey string) ([]parsers.CsvRow, []string, map[string]uint, error) {
 	parser := GetParser(parserKey)
 	if parser == nil {
-		return nil, nil, errors.New("invalid parser key")
+		return nil, nil, nil, errors.New("invalid parser key")
 	}
 	parser.InitializeParsingGroups()
 
 	// Get data for each address
 	var headers []string
 	var csvRows []parsers.CsvRow
+
+	addressRowsCount := make(map[string]uint)
+
 	for _, address := range addresses {
 		taxableTxs, err := db.GetTaxableTransactions(address, pgSQL)
 		if err != nil {
 			config.Log.Error("Error getting taxable transaction.", err)
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		// Some TXs may have fees while the address had no taxable TXs
 		// We gather all fees and pass them to the parser
 		taxableFees, err := db.GetTaxableFees(address, pgSQL)
 		if err != nil {
 			config.Log.Error("Error getting taxable fees.", err)
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		err = parser.ProcessTaxableTx(address, taxableTxs, taxableFees)
 		if err != nil {
 			config.Log.Error("Error processing taxable transaction.", err)
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		taxableEvents, err := db.GetTaxableEvents(address, pgSQL)
 		if err != nil {
 			config.Log.Error("Error getting taxable events.", err)
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		err = parser.ProcessTaxableEvent(taxableEvents)
 		if err != nil {
 			config.Log.Error("Error processing taxable events.", err)
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		// Get rows once right at the end, also filter them by date
 		rows, err := parser.GetRows(address, startDate, endDate)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 
 		csvRows = append(csvRows, rows...)
+		addressRowsCount[address] = uint(len(rows))
 		headers = parser.GetHeaders()
 	}
 	// re-sort rows if needed
 	if len(addresses) > 1 {
 		SortRows(csvRows, parser.TimeLayout())
 	}
-	return csvRows, headers, nil
+	return csvRows, headers, addressRowsCount, nil
 }
 
 func SortRows(csvRows []parsers.CsvRow, timeLayout string) {

test/indexer_test.go

@@ -24,7 +24,7 @@ func TestOsmosisCsvForAddress(t *testing.T) {
 	addressPrefix := OsmosisAddressPrefix
 	gorm, _ := dbSetup(addressRegex, addressPrefix)
 	address := "osmo14mmus5h7m6vkp0pteks8wawaj4wf3sx7fy3s2r" // local test key address
-	csvRows, headers, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing")
+	csvRows, headers, _, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing")
 	if err != nil || len(csvRows) == 0 {
 		t.Fatal("Failed to lookup taxable events")
 	}
@@ -51,7 +51,7 @@ func TestCsvForAddress(t *testing.T) {
 	// address := "juno1mt72y3jny20456k247tc5gf2dnat76l4ynvqwl"
 	// address := "juno130mdu9a0etmeuw52qfxk73pn0ga6gawk4k539x" // strangelove's delegator
 	address := "juno1m2hg5t7n8f6kzh8kmh98phenk8a4xp5wyuz34y" // local test key address
-	csvRows, headers, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing")
+	csvRows, headers, _, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing")
 	if err != nil || len(csvRows) == 0 {
 		t.Fatal("Failed to lookup taxable events")
 	}