diff --git a/src/config/config.go b/src/config/config.go index 60a3061..44c9ba0 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -96,7 +96,7 @@ type configType struct { // Routines RoutinesRunOnly bool `envconfig:"ROUTINES_RUN_ONLY" required:"false" default:"false"` RoutinesSleepDuration time.Duration `envconfig:"ROUTINES_SLEEP_DURATION" required:"false" default:"1h"` - RoutinesBatchSize int `envconfig:"ROUTINES_BATCH_SIZE" required:"false" default:"1000"` + RoutinesBatchSize int `envconfig:"ROUTINES_BATCH_SIZE" required:"false" default:"10"` RoutinesNumWorkers int `envconfig:"ROUTINES_NUM_WORKERS" required:"false" default:"1"` // FindMissing @@ -106,7 +106,10 @@ type configType struct { FindMissingExtractorAPILocation string `envconfig:"FIND_MISSING_EXTRACTOR_API_LOCATION" required:"false" default:"http://localhost:8000/api/v1"` // Redis Recovery - RedisRecoveryRunOnly bool `envconfig:"REDIS_RECOVERY_RUN_ONLY" required:"false" default:"false"` + RedisRecoveryRunOnly bool `envconfig:"REDIS_RECOVERY_RUN_ONLY" required:"false" default:"false"` + RedisRecoveryContractsOnly bool `envconfig:"REDIS_RECOVERY_CONTRACTS_ONLY" required:"false" default:"false"` + RedisRecoveryAddresses bool `envconfig:"REDIS_RECOVERY_ADDRESSES" required:"false" default:"true"` + RedisRecoveryTokenAddresses bool `envconfig:"REDIS_RECOVERY_TOKEN_ADDRESSES" required:"false" default:"true"` // Backfill Params ProcessCounts bool `envconfig:"TRANSFORMER_PROCESS_COUNTS" required:"false" default:"true"` diff --git a/src/crud/crud_base_select.go b/src/crud/crud_base_select.go index 035e4be..ef26415 100644 --- a/src/crud/crud_base_select.go +++ b/src/crud/crud_base_select.go @@ -102,6 +102,7 @@ func (m *Crud[Model, ModelOrm]) SelectBatchOrder( limit int, skip int, order string, + only_contracts bool, ) (*[]Model, error) { db := m.db db = db.Model(&m.Model) @@ -110,6 +111,9 @@ func (m *Crud[Model, ModelOrm]) SelectBatchOrder( db = db.Offset(skip) } db = db.Order(order) + if only_contracts { + db = db.Where("is_contract = true") + } output := &[]Model{} db = db.Find(output) return output, db.Error diff --git a/src/routines/cron.go b/src/routines/cron.go index 9ad7594..a912087 100644 --- a/src/routines/cron.go +++ b/src/routines/cron.go @@ -10,6 +10,8 @@ var cronRoutines = []func(){ addressIsPrep, tokenAddressCountRoutine, // Isn't used - RM? countAddressesToRedisRoutine, + setTransactionCounts, + countAddressesToRedisRoutine, } func CronStart() { diff --git a/src/routines/routines.go b/src/routines/routines.go index 9e88f64..05a7df3 100644 --- a/src/routines/routines.go +++ b/src/routines/routines.go @@ -27,20 +27,24 @@ var tokenAddressRoutines = []func(t *models.TokenAddress){ func StartRecovery() { zap.S().Warn("Init recovery...") - - //Global count - setTransactionCounts() - // One shot if config.Config.NetworkName == "mainnet" { addressTypeRoutine() } - countAddressesToRedisRoutine() + + // Moved to cron + //Global count + //setTransactionCounts() + //countAddressesToRedisRoutine() // By address - LoopRoutine(crud.GetCrud(models.Address{}, models.AddressORM{}), addressRoutines) + if config.Config.RedisRecoveryAddresses { + LoopRoutine(crud.GetCrud(models.Address{}, models.AddressORM{}), addressRoutines) + } // By token address - LoopRoutine(crud.GetCrud(models.TokenAddress{}, models.TokenAddressORM{}), tokenAddressRoutines) + if config.Config.RedisRecoveryTokenAddresses { + LoopRoutine(crud.GetCrud(models.TokenAddress{}, models.TokenAddressORM{}), tokenAddressRoutines) + } zap.S().Info("finished recovery. exiting..") time.Sleep(30 * time.Second) @@ -55,7 +59,8 @@ func LoopRoutine[M any, O any](Crud *crud.Crud[M, O], routines []func(*M)) { zap.S().Info("Starting worker on table=", Crud.TableName, " with skip=", skip, " with limit=", limit) // Run loop until addresses have all been iterated over for { - routineItems, err := Crud.SelectBatchOrder(limit, skip, "address") + // Grabs a set of addresses or just contracts + routineItems, err := Crud.SelectBatchOrder(limit, skip, "address", config.Config.RedisRecoveryContractsOnly) if errors.Is(err, gorm.ErrRecordNotFound) { zap.S().Warn("Ending address routine with error=", err.Error()) break