From a0304e4e751c6da5f0d1c2b9ccafd1486e35ecc5 Mon Sep 17 00:00:00 2001 From: Sergey <83376337+freak12techno@users.noreply.github.com> Date: Sun, 1 Oct 2023 17:56:38 +0300 Subject: [PATCH] feat: refactor historical validators pagination (#29) --- config.example.toml | 10 ---------- pkg/config/chain.go | 7 +++---- pkg/tendermint/http.go | 14 +++++++++++--- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/config.example.toml b/config.example.toml index f4d5b6a..e5a7da5 100644 --- a/config.example.toml +++ b/config.example.toml @@ -85,16 +85,6 @@ slashing-params = 300 slashing-params = 300 # Queries pagination params. [[chains.pagination]] -# How many historical validators to query at once. Why you might need this: -# When querying for historical validators on chains with more than 100 validators, you need to do multiple queries, -# as Tendermint returns 100 validators at max. If your chain has exactly 100 validators, querying first batch -# would work, but second one won't with the following error: "page should be within [1, 1] range, given 2" -# meaning that asking for a second batch of validators of 100 entries on a chain with 100 validators -# won't work. If your chain has 100 validators, consider setting it to something like 95, so first batch -# would return 95 validators, and the second one would return 5 validators. -# Do not try setting it to more than 100, this won't work. -# Defaults to 100. -historical-validators = 95 # How many blocks to query at once. # Defaults to 100. # Decrease it if the node you're querying has rate limiting and doing too many requests at once diff --git a/pkg/config/chain.go b/pkg/config/chain.go index 77ff62b..0dbea22 100644 --- a/pkg/config/chain.go +++ b/pkg/config/chain.go @@ -7,10 +7,9 @@ import ( ) type ChainPagination struct { - HistoricalValidators int `default:"100" toml:"historical-validators"` - BlocksSearch int `default:"100" toml:"blocks-search"` - ValidatorsList uint64 `default:"1000" toml:"validators-list"` - SigningInfos uint64 `default:"1000" toml:"signing-infos"` + BlocksSearch int `default:"100" toml:"blocks-search"` + ValidatorsList uint64 `default:"1000" toml:"validators-list"` + SigningInfos uint64 `default:"1000" toml:"signing-infos"` } type ChainConfig struct { Name string `toml:"name"` diff --git a/pkg/tendermint/http.go b/pkg/tendermint/http.go index dadd196..fc9f20c 100644 --- a/pkg/tendermint/http.go +++ b/pkg/tendermint/http.go @@ -10,6 +10,7 @@ import ( "main/pkg/utils" "net/http" "net/url" + "strconv" "strings" "time" @@ -239,9 +240,8 @@ func (rpc *RPC) GetActiveSetAtBlock(height int64) (map[string]bool, error) { for { queryURL := fmt.Sprintf( - "/validators?height=%d&per_page=%d&page=%d", + "/validators?height=%d&per_page=100&page=%d", height, - rpc.config.Pagination.HistoricalValidators, page, ) @@ -265,11 +265,19 @@ func (rpc *RPC) GetActiveSetAtBlock(height int64) (map[string]bool, error) { return nil, err } + validatorsCount, err := strconv.Atoi(response.Result.Total) + if err != nil { + rpc.logger.Warn(). + Err(err). + Msg("Error parsing validators count from response") + return nil, err + } + for _, validator := range response.Result.Validators { activeSetMap[validator.Address] = true } - if len(response.Result.Validators) < rpc.config.Pagination.HistoricalValidators { + if len(activeSetMap) >= validatorsCount { break }