diff --git a/client/client.go b/client/client.go index fb16327..a80efba 100644 --- a/client/client.go +++ b/client/client.go @@ -101,6 +101,12 @@ func setup() (*gorm.DB, *config.ClientConfig, int, string, error) { dbTypes.CacheDenoms(db) dbTypes.CacheIBCDenoms(db) + err = db.AutoMigrate(&dbTypes.Address{}, &AddressUsageCSV{}, &AddressUsageJSON{}) + + if err != nil { + config.Log.Fatalf("Error migrating client models. Err: %v", err) + } + return db, &cfg, svcPort, cfg.Client.Model, nil } @@ -258,15 +264,37 @@ func GetTaxableEventsCSV(c *gin.Context) { return } - accountRows, headers, err := csv.ParseForAddress(addresses, startDate, endDate, DB, format) + // We only want to process and return data on addresses we know are valid (in our DB) + validAddresses, validAddressDBObjects, err := GetValidAddresses(addresses) + if err != nil { + config.Log.Errorf("Error getting valid addresses: %v", addresses) + c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck + return + } + + accountRows, headers, addressRowsCount, err := csv.ParseForAddress(validAddresses, startDate, endDate, DB, format) if err != nil { // the error returned here has already been pushed to the context... I think. - config.Log.Errorf("Error getting rows for addresses: %v", addresses) + config.Log.Errorf("Error getting valid addresses %v: %s", addresses, err) fmt.Println(err) c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck return } + for _, address := range validAddressDBObjects { + usage := AddressUsageCSV{ + Address: address, + AddressID: address.ID, + RowsRetrieved: addressRowsCount[address.Address], + Timestamp: time.Now(), + } + + err = DB.Create(&usage).Error + if err != nil { + config.Log.Errorf("Error saving address usage: %v", err) + } + } + if len(accountRows) == 0 { c.JSON(404, gin.H{"message": "No transactions for given address"}) return @@ -306,14 +334,36 @@ func GetTaxableEventsJSON(c *gin.Context) { return } - accountRows, _, err := csv.ParseForAddress(addresses, startDate, endDate, DB, format) + // We only want to process and return data on addresses we know are valid (in our DB) + validAddresses, validAddressDBObjects, err := GetValidAddresses(addresses) + if err != nil { + config.Log.Errorf("Error getting valid addresses %v: %s", addresses, err) + c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck + return + } + + accountRows, _, addressRowsCount, err := csv.ParseForAddress(validAddresses, startDate, endDate, DB, format) if err != nil { // the error returned here has already been pushed to the context... I think. - config.Log.Errorf("Error getting rows for addresses: %v", addresses) + config.Log.Errorf("Error getting rows for addresses: %v", validAddresses) c.AbortWithError(500, errors.New("error getting rows for address")) // nolint:staticcheck,errcheck return } + for _, address := range validAddressDBObjects { + usage := AddressUsageJSON{ + Address: address, + AddressID: address.ID, + RowsRetrieved: addressRowsCount[address.Address], + Timestamp: time.Now(), + } + + err = DB.Create(&usage).Error + if err != nil { + config.Log.Errorf("Error saving address usage: %v", err) + } + } + if len(accountRows) == 0 { c.JSON(404, gin.H{"message": "No transactions for given address"}) return @@ -374,3 +424,49 @@ func ParseTaxableEventsBody(c *gin.Context) ([]string, string, *time.Time, *time return addresses, format, startDate, endDate, nil } + +func GetValidAddresses(addresses []string) ([]string, []dbTypes.Address, error) { + validAddresses, err := dbTypes.GetAddresses(addresses, DB) + if err != nil { + config.Log.Errorf("Error getting addresses: %v", addresses) + return nil, nil, err + } + + var validAddressesStr []string + if len(validAddresses) != len(addresses) { + for _, address := range addresses { + found := false + for _, validAddress := range validAddresses { + if address == validAddress.Address { + found = true + break + } + } + if found { + validAddressesStr = append(validAddressesStr, address) + } else { + config.Log.Infof("Invalid address not found in DB, skipping: %v", address) + } + } + } else { + validAddressesStr = addresses + } + + return validAddressesStr, validAddresses, nil +} + +type AddressUsageCSV struct { + ID uint + Address dbTypes.Address + AddressID uint + RowsRetrieved uint + Timestamp time.Time +} + +type AddressUsageJSON struct { + ID uint + Address dbTypes.Address + AddressID uint + RowsRetrieved uint + Timestamp time.Time +} diff --git a/cmd/query.go b/cmd/query.go index bf8dfa4..89c6521 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -50,7 +50,7 @@ var queryCmd = &cobra.Command{ endDate = &parsedDate } - csvRows, headers, err := csv.ParseForAddress(queryConfig.Base.Addresses, startDate, endDate, db, queryConfig.Base.Format) + csvRows, headers, _, err := csv.ParseForAddress(queryConfig.Base.Addresses, startDate, endDate, db, queryConfig.Base.Format) if err != nil { log.Println(queryConfig.Base.Addresses) config.Log.Fatal("Error calling parser for address", err) diff --git a/csv/parser.go b/csv/parser.go index f3b610e..b115371 100644 --- a/csv/parser.go +++ b/csv/parser.go @@ -45,21 +45,24 @@ func GetParser(parserKey string) parsers.Parser { return nil } -func ParseForAddress(addresses []string, startDate, endDate *time.Time, pgSQL *gorm.DB, parserKey string) ([]parsers.CsvRow, []string, error) { +func ParseForAddress(addresses []string, startDate, endDate *time.Time, pgSQL *gorm.DB, parserKey string) ([]parsers.CsvRow, []string, map[string]uint, error) { parser := GetParser(parserKey) if parser == nil { - return nil, nil, errors.New("invalid parser key") + return nil, nil, nil, errors.New("invalid parser key") } parser.InitializeParsingGroups() // Get data for each address var headers []string var csvRows []parsers.CsvRow + + addressRowsCount := make(map[string]uint) + for _, address := range addresses { taxableTxs, err := db.GetTaxableTransactions(address, pgSQL) if err != nil { config.Log.Error("Error getting taxable transaction.", err) - return nil, nil, err + return nil, nil, nil, err } // Some TXs may have fees while the address had no taxable TXs @@ -67,41 +70,42 @@ func ParseForAddress(addresses []string, startDate, endDate *time.Time, pgSQL *g taxableFees, err := db.GetTaxableFees(address, pgSQL) if err != nil { config.Log.Error("Error getting taxable fees.", err) - return nil, nil, err + return nil, nil, nil, err } err = parser.ProcessTaxableTx(address, taxableTxs, taxableFees) if err != nil { config.Log.Error("Error processing taxable transaction.", err) - return nil, nil, err + return nil, nil, nil, err } taxableEvents, err := db.GetTaxableEvents(address, pgSQL) if err != nil { config.Log.Error("Error getting taxable events.", err) - return nil, nil, err + return nil, nil, nil, err } err = parser.ProcessTaxableEvent(taxableEvents) if err != nil { config.Log.Error("Error processing taxable events.", err) - return nil, nil, err + return nil, nil, nil, err } // Get rows once right at the end, also filter them by date rows, err := parser.GetRows(address, startDate, endDate) if err != nil { - return nil, nil, err + return nil, nil, nil, err } csvRows = append(csvRows, rows...) + addressRowsCount[address] = uint(len(rows)) headers = parser.GetHeaders() } // re-sort rows if needed if len(addresses) > 1 { SortRows(csvRows, parser.TimeLayout()) } - return csvRows, headers, nil + return csvRows, headers, addressRowsCount, nil } func SortRows(csvRows []parsers.CsvRow, timeLayout string) { diff --git a/test/indexer_test.go b/test/indexer_test.go index 61d4f52..5cbec46 100644 --- a/test/indexer_test.go +++ b/test/indexer_test.go @@ -24,7 +24,7 @@ func TestOsmosisCsvForAddress(t *testing.T) { addressPrefix := OsmosisAddressPrefix gorm, _ := dbSetup(addressRegex, addressPrefix) address := "osmo14mmus5h7m6vkp0pteks8wawaj4wf3sx7fy3s2r" // local test key address - csvRows, headers, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing") + csvRows, headers, _, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing") if err != nil || len(csvRows) == 0 { t.Fatal("Failed to lookup taxable events") } @@ -51,7 +51,7 @@ func TestCsvForAddress(t *testing.T) { // address := "juno1mt72y3jny20456k247tc5gf2dnat76l4ynvqwl" // address := "juno130mdu9a0etmeuw52qfxk73pn0ga6gawk4k539x" // strangelove's delegator address := "juno1m2hg5t7n8f6kzh8kmh98phenk8a4xp5wyuz34y" // local test key address - csvRows, headers, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing") + csvRows, headers, _, err := csv.ParseForAddress([]string{address}, nil, nil, gorm, "accointing") if err != nil || len(csvRows) == 0 { t.Fatal("Failed to lookup taxable events") }