feat: add bulk query config params (#6)

config.example.toml

@@ -66,7 +66,9 @@ min-signed-per-window = 0.05
 # will be overriden on chain startup with the actual values from chain, and will be periodically updated.
 # Defaults to true
 query-slashing-params = true
-# Pagination params. Why you might need this:
+# Queries pagination params.
+[[chains.pagination]]
+# How many historical validators to query at once. Why you might need this:
 # When querying for historical validators on chains with more than 100 validators, you need to do multiple queries,
 # as Tendermint returns 100 validators at max. If your chain has exactly 100 validators, querying first batch
 # would work, but second one won't with the following error: "page should be within [1, 1] range, given 2"
@@ -75,7 +77,16 @@ query-slashing-params = true
 # would return 95 validators, and the second one would return 5 validators.
 # Do not try setting it to more than 100, this won't work.
 # Defaults to 100.
-pagination = { historical-validators = 95 }
+historical-validators = 95
+# How many blocks to query at once.
+# Defaults to 100.
+# Decrease it if the node you're querying has rate limiting and doing too many requests at once
+# causes some requests to fail
+blocks-search = 100
+# How many validators to query at once.
+validators-list = 1000
+# How many signing infos to query at once.
+validators-list = 1000
 
 # You can specify multiple chain. Each chain should have its own set of reporters
 # and they should not overlap.

pkg/app_manager.go

@@ -3,7 +3,6 @@ package pkg
 import (
 	"fmt"
 	configPkg "main/pkg/config"
-	"main/pkg/constants"
 	dataPkg "main/pkg/data"
 	databasePkg "main/pkg/database"
 	"main/pkg/metrics"
@@ -384,7 +383,7 @@ func (a *AppManager) PopulateBlocks() {
 		return
 	}
 
-	blocksChunks := utils.SplitIntoChunks(missingBlocks, int(constants.BlockSearchPagination))
+	blocksChunks := utils.SplitIntoChunks(missingBlocks, a.Config.Pagination.BlocksSearch)
 
 	for _, chunk := range blocksChunks {
 		count := a.StateManager.GetBlocksCountSinceLatest(a.Config.StoreBlocks)

pkg/config/chain.go

@@ -7,9 +7,11 @@ import (
 )
 
 type ChainPagination struct {
-	HistoricalValidators int `default:"100"  toml:"historical-validators"`
+	HistoricalValidators int    `default:"100"  toml:"historical-validators"`
+	BlocksSearch         int    `default:"100"  toml:"blocks-search"`
+	ValidatorsList       uint64 `default:"1000" toml:"validators-list"`
+	SigningInfos         uint64 `default:"1000" toml:"signing-infos"`
 }
-
 type ChainConfig struct {
 	Name                 string          `toml:"name"`
 	PrettyName           string          `toml:"pretty-name"`

pkg/constants/constants.go

@@ -6,10 +6,6 @@ type QueryType string
 type FormatType string
 
 const (
-	BlockSearchPagination       int64  = 100
-	ValidatorsQueryPagination   uint64 = 1000
-	SigningInfosQueryPagination uint64 = 1000
-
 	NewBlocksQuery = "tm.event='NewBlock'"
 
 	ValidatorBonded int32 = 3

pkg/tendermint/http.go

@@ -130,7 +130,7 @@ func (rpc *RPC) AbciQuery(
 func (rpc *RPC) GetValidators(height int64) (*stakingTypes.QueryValidatorsResponse, error) {
 	query := stakingTypes.QueryValidatorsRequest{
 		Pagination: &queryTypes.PageRequest{
-			Limit: constants.ValidatorsQueryPagination,
+			Limit: rpc.config.Pagination.ValidatorsList,
 		},
 	}
 
@@ -152,7 +152,7 @@ func (rpc *RPC) GetValidators(height int64) (*stakingTypes.QueryValidatorsRespon
 func (rpc *RPC) GetSigningInfos(height int64) (*slashingTypes.QuerySigningInfosResponse, error) {
 	query := slashingTypes.QuerySigningInfosRequest{
 		Pagination: &queryTypes.PageRequest{
-			Limit: constants.SigningInfosQueryPagination,
+			Limit: rpc.config.Pagination.SigningInfos,
 		},
 	}