diff --git a/abci/abci.go b/abci/abci.go index 9e786ed..7f4609f 100644 --- a/abci/abci.go +++ b/abci/abci.go @@ -8,8 +8,9 @@ import ( "net/http" ) -func LastBlockHeight() (int64, error) { - url := `https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org/abci_info?last_block_height` +func LastBlockHeight(rpc string) (int64, error) { + // https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org/abci_info?last_block_height + url := fmt.Sprintf("%v/abci_info?last_block_height", rpc) resp, err := http.Get(url) if err != nil { return 0, err @@ -23,7 +24,6 @@ func LastBlockHeight() (int64, error) { func BsDBInfoBlockHeight(spHost string, height int64) (string, error) { // https://gnfd-testnet-sp1.bnbchain.org/?bsdb-info&block_height=11037600 url := fmt.Sprintf("https://%v/?bsdb-info&block_height=%v", spHost, height) - //fmt.Println(url) resp, err := http.Get(url) if err != nil { return "", err diff --git a/checks/CheckSpDbObjectNumsPeriod.go b/checks/CheckSpDbObjectNumsPeriod.go deleted file mode 100644 index 1f8ec14..0000000 --- a/checks/CheckSpDbObjectNumsPeriod.go +++ /dev/null @@ -1,108 +0,0 @@ -package checks - -import ( - "fmt" - "github.com/bnb-chain/gnfd-qa-test-monitor/abci" - "github.com/bnb-chain/gnfd-qa-test-monitor/utils" - "github.com/tidwall/gjson" -) - -var SelfMainNetSpHost = []string{ - "greenfield-sp.bnbchain.org", - "greenfield-sp.defibit.io", - "greenfield-sp.ninicoin.io", - "greenfield-sp.nariox.org", - "greenfield-sp.lumibot.org", - "greenfield-sp.voltbot.io", - "greenfield-sp.nodereal.io", -} - -var SelfTestNetSpHost = []string{ - "gnfd-testnet-sp1.bnbchain.org", - "gnfd-testnet-sp2.bnbchain.org", - "gnfd-testnet-sp3.bnbchain.org", - "gnfd-testnet-sp4.bnbchain.org", - "gnfd-testnet-sp1.nodereal.io", - "gnfd-testnet-sp2.nodereal.io", - "gnfd-testnet-sp3.nodereal.io", -} - -type Code float64 - -const ( - OK Code = iota - GetObjectTotalCountErr - GetObjectSealCountErr - CheckObjectTotalCountErr - CheckObjectSealCountErr -) - -func CheckSpDbObjectNumsPeriod(spHostArray []string) (block int64, errCode Code) { - chainHeight, err := abci.LastBlockHeight() - if err != nil { - fmt.Println(err) - } - fmt.Printf("height: %d\n", chainHeight) - calcHeight := chainHeight / 3600 * 3600 - fmt.Printf("calcHeight: %d\n", calcHeight) - - resObjectCount := make(map[string][]gjson.Result) - resObjectSealCount := make(map[string][]gjson.Result) - - // get everyone sp object count - for _, spHost := range spHostArray { - xmlResult, err := abci.BsDBInfoBlockHeight(spHost, calcHeight) - if err != nil { - fmt.Println(err) - } - - objectResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectTotalCount") - if objectResString == "" { - fmt.Printf("sp: %v, ObjectTotalCount error\n", spHost) - return calcHeight, GetObjectTotalCountErr - } else { - ObjectTotalCount := gjson.Parse(objectResString).Array() - resObjectCount[spHost] = ObjectTotalCount - } - - objectSealResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectSealCount") - if objectSealResString == "" { - fmt.Printf("sp: %v, ObjectSealCount error\n", spHost) - return calcHeight, GetObjectSealCountErr - } else { - ObjectSealCount := gjson.Parse(objectSealResString).Array() - resObjectSealCount[spHost] = ObjectSealCount - } - } - - // check sp object count - for i := 0; i < 64; i++ { - sumObject := int64(0) - sumSp1 := int64(0) - for _, objectCount := range resObjectCount { - sumObject = sumObject + objectCount[i].Int() - sumSp1++ - } - sumSealedObject := int64(0) - sumSp2 := int64(0) - for _, sealObjectCount := range resObjectSealCount { - sumSealedObject = sumSealedObject + sealObjectCount[i].Int() - sumSp2++ - } - - objectAverage := sumObject / sumSp1 - sealObjectAverage := sumSealedObject / sumSp2 - for _, eachValue := range resObjectCount { - if objectAverage != eachValue[i].Int() { - return calcHeight, CheckObjectTotalCountErr - } - } - for _, eachValue := range resObjectSealCount { - if sealObjectAverage != eachValue[i].Int() { - return calcHeight, CheckObjectSealCountErr - } - } - } - - return calcHeight, OK -} diff --git a/checks/dbshard.go b/checks/dbshard.go new file mode 100644 index 0000000..9e23eff --- /dev/null +++ b/checks/dbshard.go @@ -0,0 +1,165 @@ +package checks + +import ( + "fmt" + "github.com/bnb-chain/gnfd-qa-test-monitor/abci" + "github.com/bnb-chain/gnfd-qa-test-monitor/utils" + "github.com/tidwall/gjson" +) + +var MainNetSpHost = []string{ + "greenfield-sp.bnbchain.org", + "greenfield-sp.defibit.io", + "greenfield-sp.ninicoin.io", + "greenfield-sp.nariox.org", + "greenfield-sp.lumibot.org", + "greenfield-sp.voltbot.io", + "greenfield-sp.nodereal.io", +} + +var TestNetSpHost = []string{ + "gnfd-testnet-sp1.bnbchain.org", + "gnfd-testnet-sp2.bnbchain.org", + "gnfd-testnet-sp3.bnbchain.org", + "gnfd-testnet-sp4.bnbchain.org", + "gnfd-testnet-sp1.nodereal.io", + "gnfd-testnet-sp2.nodereal.io", + "gnfd-testnet-sp3.nodereal.io", +} + +type Code uint32 + +const ( + OK Code = iota + GetObjectTotalCountErr + GetObjectSealCountErr + CheckObjectTotalCountErr + CheckObjectSealCountErr +) + +func CheckDbShard(checkHeight int64, spHost string) Code { + xmlResult, err := abci.BsDBInfoBlockHeight(spHost, checkHeight) + if err != nil { + fmt.Println(err) + } + + var resObjectCount, resObjectSealCount []gjson.Result + objectResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectTotalCount") + if objectResString == "" { + fmt.Printf("sp: %v, ObjectTotalCount error\n", spHost) + return GetObjectTotalCountErr + } else { + objectTotalCount := gjson.Parse(objectResString).Array() + resObjectCount = objectTotalCount + } + + objectSealResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectSealCount") + if objectSealResString == "" { + fmt.Printf("sp: %v, ObjectSealCount error\n", spHost) + return GetObjectSealCountErr + } else { + ObjectSealCount := gjson.Parse(objectSealResString).Array() + resObjectSealCount = ObjectSealCount + } + + // check sp object count + for i := 0; i < 64; i++ { + sumObject := int64(0) + sumSp1 := int64(0) + for _, objectCount := range resObjectCount { + sumObject = sumObject + objectCount.Int() + sumSp1++ + } + sumSealedObject := int64(0) + sumSp2 := int64(0) + for _, sealObjectCount := range resObjectSealCount { + sumSealedObject = sumSealedObject + sealObjectCount.Int() + sumSp2++ + } + + objectAverage := sumObject / sumSp1 + sealObjectAverage := sumSealedObject / sumSp2 + for _, eachValue := range resObjectCount { + if objectAverage != eachValue.Int() { + return CheckObjectTotalCountErr + } + } + for _, eachValue := range resObjectSealCount { + if sealObjectAverage != eachValue.Int() { + return CheckObjectSealCountErr + } + } + } + + return OK +} + +//func CheckSpDbObjectNumsPeriod1(spHostArray []string) (block int64, errCode Code) { +// chainHeight, err := abci.LastBlockHeight() +// if err != nil { +// fmt.Println(err) +// } +// fmt.Printf("height: %d\n", chainHeight) +// calcHeight := chainHeight / 3600 * 3600 +// fmt.Printf("calcHeight: %d\n", calcHeight) +// +// resObjectCount := make(map[string][]gjson.Result) +// resObjectSealCount := make(map[string][]gjson.Result) +// +// // get everyone sp object count +// for _, spHost := range spHostArray { +// xmlResult, err := abci.BsDBInfoBlockHeight(spHost, calcHeight) +// if err != nil { +// fmt.Println(err) +// } +// +// objectResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectTotalCount") +// if objectResString == "" { +// fmt.Printf("sp: %v, ObjectTotalCount error\n", spHost) +// return calcHeight, GetObjectTotalCountErr +// } else { +// ObjectTotalCount := gjson.Parse(objectResString).Array() +// resObjectCount[spHost] = ObjectTotalCount +// } +// +// objectSealResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectSealCount") +// if objectSealResString == "" { +// fmt.Printf("sp: %v, ObjectSealCount error\n", spHost) +// return calcHeight, GetObjectSealCountErr +// } else { +// ObjectSealCount := gjson.Parse(objectSealResString).Array() +// resObjectSealCount[spHost] = ObjectSealCount +// } +// } +// +// // check sp object count +// for i := 0; i < 64; i++ { +// sumObject := int64(0) +// sumSp1 := int64(0) +// for _, objectCount := range resObjectCount { +// sumObject = sumObject + objectCount[i].Int() +// sumSp1++ +// } +// sumSealedObject := int64(0) +// sumSp2 := int64(0) +// for _, sealObjectCount := range resObjectSealCount { +// sumSealedObject = sumSealedObject + sealObjectCount[i].Int() +// sumSp2++ +// } +// +// objectAverage := sumObject / sumSp1 +// sealObjectAverage := sumSealedObject / sumSp2 +// for _, eachValue := range resObjectCount { +// if objectAverage != eachValue[i].Int() { +// return calcHeight, CheckObjectTotalCountErr +// } +// } +// for _, eachValue := range resObjectSealCount { +// if sealObjectAverage != eachValue[i].Int() { +// return calcHeight, CheckObjectSealCountErr +// } +// } +// } +// +// return calcHeight, OK +//} diff --git a/main.go b/main.go index 18f19e0..96bdec7 100644 --- a/main.go +++ b/main.go @@ -2,28 +2,67 @@ package main import ( "fmt" + "github.com/bnb-chain/gnfd-qa-test-monitor/abci" "github.com/bnb-chain/gnfd-qa-test-monitor/checks" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" + "strings" "time" ) func recordMetrics() { - checkBlock := promauto.NewGauge(prometheus.GaugeOpts{ - Name: "sp_db_object_nums_period_check_block_height", - }) - checkErrCode := promauto.NewGauge(prometheus.GaugeOpts{ - Name: "sp_db_object_nums_period_check_error_code", - }) + testNetRpc := "https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org:443" + mainNetRpc := "https://greenfield-chain.bnbchain.org:443" + + checkTestNetBlock := promauto.NewGauge(prometheus.GaugeOpts{Name: "testnet_sp_db_shard_check_block_height"}) + checkMainNetBlock := promauto.NewGauge(prometheus.GaugeOpts{Name: "mainnet_sp_db_shard_check_block_height"}) + + checkTestNetSpErrCodes := make([]prometheus.Gauge, len(checks.TestNetSpHost)) + for i, spHost := range checks.TestNetSpHost { + spHost = strings.Replace(spHost, "-", "_", -1) + spHost = strings.Replace(spHost, ".", "_", -1) + name := fmt.Sprintf("testnet_sp_db_shard_error_code_%v", spHost) + checkTestNetSpErrCodes[i] = promauto.NewGauge(prometheus.GaugeOpts{Name: name}) + } + + checkMainNetSpErrCodes := make([]prometheus.Gauge, len(checks.TestNetSpHost)) + for i, spHost := range checks.TestNetSpHost { + spHost = strings.Replace(spHost, "-", "_", -1) + spHost = strings.Replace(spHost, ".", "_", -1) + name := fmt.Sprintf("mainnet_sp_db_shard_error_code_%v", spHost) + checkMainNetSpErrCodes[i] = promauto.NewGauge(prometheus.GaugeOpts{Name: name}) + } go func() { for { - block, result := checks.CheckSpDbObjectNumsPeriod(checks.SelfTestNetSpHost) - fmt.Printf("block: %v, result: %v \n", block, result) - checkBlock.Set(float64(block)) - checkErrCode.Set(float64(result)) + // check TestNet + testNetChainHeight, err := abci.LastBlockHeight(testNetRpc) + if err != nil { + fmt.Println(err) + continue + } + testNetCalcHeight := testNetChainHeight / 3600 * 3600 + checkTestNetBlock.Set(float64(testNetCalcHeight)) + for i, spHost := range checks.TestNetSpHost { + errCode := checks.CheckDbShard(testNetCalcHeight, spHost) + checkTestNetSpErrCodes[i].Set(float64(errCode)) + } + + // check MainNet + mainNetChainHeight, err := abci.LastBlockHeight(mainNetRpc) + if err != nil { + fmt.Println(err) + continue + } + mainNetCalcHeight := mainNetChainHeight / 3600 * 3600 + checkMainNetBlock.Set(float64(mainNetCalcHeight)) + for i, spHost := range checks.MainNetSpHost { + errCode := checks.CheckDbShard(mainNetCalcHeight, spHost) + checkMainNetSpErrCodes[i].Set(float64(errCode)) + } + time.Sleep(time.Minute * 10) } }()