From fda725da8e01e688bb6016b27de1e0a78d2245b0 Mon Sep 17 00:00:00 2001 From: longtao <1273211023@qq.com> Date: Mon, 3 Apr 2023 18:49:41 +0800 Subject: [PATCH 1/4] use ccdata when api.coingecko access errors --- configs/cfg.toml | 1 + internal/app/conf/config.go | 1 + internal/app/task/denom_heatmap.go | 86 +++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/configs/cfg.toml b/configs/cfg.toml index e19b4d8..53c2515 100644 --- a/configs/cfg.toml +++ b/configs/cfg.toml @@ -39,6 +39,7 @@ db = 0 [spi] coingecko_price_url = "https://api.coingecko.com/api/v3/simple/price" +ccdata_price_url = "https://min-api.cryptocompare.com/data/pricemulti?api_key=f58cdd6f5d9cf0f6c0f44a1036924e2b17391ef3aea75e2be5f6133ff4c1ed84" [task] cron_time_statistic_task = 5 diff --git a/internal/app/conf/config.go b/internal/app/conf/config.go index 8c14413..f4a5f28 100644 --- a/internal/app/conf/config.go +++ b/internal/app/conf/config.go @@ -107,6 +107,7 @@ type Task struct { type Spi struct { CoingeckoPriceUrl string `mapstructure:"coingecko_price_url"` + CcDataPriceUrl string `mapstructure:"ccdata_price_url"` } type ChainConfig struct { diff --git a/internal/app/task/denom_heatmap.go b/internal/app/task/denom_heatmap.go index 054f99e..69c580d 100644 --- a/internal/app/task/denom_heatmap.go +++ b/internal/app/task/denom_heatmap.go @@ -117,7 +117,7 @@ func (t *DenomHeatmapTask) coinPriceHandler() (map[string]float64, error) { bz, err := utils.HttpGet(url) if err != nil { logrus.Errorf("task %s get coin price error, %v", t.Name(), err) - return nil, err + return t.coinPriceHandlerBackup() } var priceResp map[string]map[string]float64 @@ -148,6 +148,90 @@ func (t *DenomHeatmapTask) coinPriceHandler() (map[string]float64, error) { return priceFloatMap, nil } +func (t *DenomHeatmapTask) coinPriceHandlerBackup() (map[string]float64, error) { + var symbols []string + coinIdSymbolMap := make(map[string]string) + for _, v := range t.authDenomList { + if v.CoinId != "" { + symbols = append(symbols, v.Symbol) + coinIdSymbolMap[v.CoinId] = v.Symbol + } + } + + if len(symbols) == 0 { + return nil, fmt.Errorf("no symbol") + } + + getPriceFunc := func(fsyms string) (map[string]map[string]float64, error) { + url := fmt.Sprintf("%s&fsyms=%s&tsyms=USD", global.Config.Spi.CcDataPriceUrl, fsyms) + bz, err := utils.HttpGet(url) + if err != nil { + logrus.Errorf("task %s get coin price from ccdata error, %v", t.Name(), err) + return nil, err + } + var symbolPriceResp map[string]map[string]float64 + err = json.Unmarshal(bz, &symbolPriceResp) + if err != nil { + logrus.Errorf("task %s get coin ccdata price error, %v", t.Name(), err) + return nil, err + } + return symbolPriceResp, nil + } + // fsyms param maxlength is 300, so we need request in two steps + segment1Resp, err := getPriceFunc(strings.Join(symbols[:len(symbols)/2], ",")) + if err != nil { + return nil, err + } + segment2Resp, err := getPriceFunc(strings.Join(symbols[len(symbols)/2:], ",")) + if err != nil { + return nil, err + } + symbolPriceResp := make(map[string]map[string]float64) + for k, v := range segment1Resp { + symbolPriceResp[k] = v + } + for k, v := range segment2Resp { + symbolPriceResp[k] = v + } + + cachePriceMap, err := tokenPriceRepo.GetAll() + if err != nil { + logrus.Errorf("task %s get coin price from cache error, %v", t.Name(), err) + return nil, err + } + priceMap := make(map[string]string, len(cachePriceMap)) + for k, v := range cachePriceMap { + if symbol, ok := coinIdSymbolMap[k]; ok { + if price, exists := symbolPriceResp[symbol]; exists { + cachePriceMap[k] = price["USD"] + + result := strconv.FormatFloat(price["USD"], 'f', 12, 64) + for strings.HasSuffix(result, "0") { + result = strings.TrimSuffix(result, "0") + } + if strings.HasSuffix(result, ".") { + result = strings.TrimSuffix(result, ".") + } + priceMap[k] = result + continue + } + } + result := strconv.FormatFloat(v, 'f', 12, 64) + for strings.HasSuffix(result, "0") { + result = strings.TrimSuffix(result, "0") + } + if strings.HasSuffix(result, ".") { + result = strings.TrimSuffix(result, ".") + } + priceMap[k] = result + } + + if err = tokenPriceRepo.BatchSet(priceMap); err != nil { + logrus.Errorf("task %s set coin price cache error, %v", t.Name(), err) + } + return cachePriceMap, nil +} + // supplyHandler Get supply of denoms, then save supply info to cache func (t *DenomHeatmapTask) supplyHandler() { wg := sync.WaitGroup{} From 472c1c66c8a843ef8211cbaca8e7a5b3f6cedf8a Mon Sep 17 00:00:00 2001 From: longtao <1273211023@qq.com> Date: Mon, 17 Apr 2023 11:43:51 +0800 Subject: [PATCH 2/4] adapter uptick --- internal/app/task/common.go | 22 ++++++++++++++++++++ internal/app/task/ibc_tx_relate_task_test.go | 13 ++++++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/app/task/common.go b/internal/app/task/common.go index 7bf0ce0..e58d357 100644 --- a/internal/app/task/common.go +++ b/internal/app/task/common.go @@ -2,6 +2,7 @@ package task import ( "encoding/json" + "fmt" "sync" "time" @@ -247,6 +248,27 @@ func parseRecvPacketTxEvents(msgIndex int, tx *entity.Tx) (dcConnection, packetA } } } + + if evt.Type == "uptick.erc20.v1.EventIBCERC20" { + var message string + var status string + for _, attr := range evt.Attributes { + switch attr.Key { + case "status": + status = attr.Value + case "message": + message = attr.Value + default: + } + } + if status == "\"STATUS_SUCCESS\"" { + packetAck = status + existPacketAck = true + } else { + packetAck = fmt.Sprintf("error:\"%s\" ", message) + existPacketAck = true + } + } } } diff --git a/internal/app/task/ibc_tx_relate_task_test.go b/internal/app/task/ibc_tx_relate_task_test.go index 046adec..6ec8c0a 100644 --- a/internal/app/task/ibc_tx_relate_task_test.go +++ b/internal/app/task/ibc_tx_relate_task_test.go @@ -1,6 +1,7 @@ package task import ( + "fmt" "github.com/bianjieai/iobscan-ibc-explorer-backend/internal/app/utils" "testing" ) @@ -26,3 +27,15 @@ func Test_HandlerIbcTxs(t *testing.T) { rw.handlerIbcTxs(chain, ibcTxList, denomMap) t.Log(utils.MustMarshalJsonToStr(ibcTxList)) } + +func Test_parseRecvPacketTxEvents(t *testing.T) { + txs, err := txRepo.GetTxByHashes("uptick", []string{"C476E603D7A3329FCB8486897B465815F10E3A6B70F7BC89657986409CBC3FB6"}) + if err != nil { + t.Log(err.Error()) + } + + dcConnection, packetAck, exists := parseRecvPacketTxEvents(1, txs[0]) + fmt.Println(dcConnection) + fmt.Println(packetAck) + fmt.Println(exists) +} From 9d0187f7d1e4bc8e21ec2d2ecb2f2451978be046 Mon Sep 17 00:00:00 2001 From: longtao <1273211023@qq.com> Date: Mon, 17 Apr 2023 11:55:25 +0800 Subject: [PATCH 3/4] adapter uptick --- internal/app/task/common.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/app/task/common.go b/internal/app/task/common.go index e58d357..06ba770 100644 --- a/internal/app/task/common.go +++ b/internal/app/task/common.go @@ -256,6 +256,7 @@ func parseRecvPacketTxEvents(msgIndex int, tx *entity.Tx) (dcConnection, packetA switch attr.Key { case "status": status = attr.Value + existPacketAck = true case "message": message = attr.Value default: @@ -263,10 +264,8 @@ func parseRecvPacketTxEvents(msgIndex int, tx *entity.Tx) (dcConnection, packetA } if status == "\"STATUS_SUCCESS\"" { packetAck = status - existPacketAck = true } else { packetAck = fmt.Sprintf("error:\"%s\" ", message) - existPacketAck = true } } } From ce18f31e15c160a37b435660dca97c594cca24cd Mon Sep 17 00:00:00 2001 From: longtao <1273211023@qq.com> Date: Fri, 19 May 2023 11:15:03 +0800 Subject: [PATCH 4/4] cache chain all day volume --- internal/app/constant/constant.go | 3 ++- .../app/task/ibc_chain_inflow_statistics.go | 26 ++++++++++++++----- .../app/task/ibc_chain_outflow_statistics.go | 26 ++++++++++++++----- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/internal/app/constant/constant.go b/internal/app/constant/constant.go index d811dd1..b6239de 100644 --- a/internal/app/constant/constant.go +++ b/internal/app/constant/constant.go @@ -62,7 +62,8 @@ const ( SECP256K1 = "secp256k1" ICS20 = "ics20" - ChainFlowTrendDays = 365 + ChainFlowTrendDays = 365 + ChainFlowVolumeDays = 10000 ExportTxsNum = 1000 ) diff --git a/internal/app/task/ibc_chain_inflow_statistics.go b/internal/app/task/ibc_chain_inflow_statistics.go index 8108b06..eb46707 100644 --- a/internal/app/task/ibc_chain_inflow_statistics.go +++ b/internal/app/task/ibc_chain_inflow_statistics.go @@ -209,6 +209,9 @@ func (t *ChainInflowStatisticsTask) setStatisticsDataCache() { startTime, _ := lastNDaysZeroTimeUnix(days) _, endTime := todayUnix() + allDays := constant.ChainFlowVolumeDays + allStartTime, _ := lastNDaysZeroTimeUnix(allDays) + chainInfosMap, err := getAllChainInfosMap() if err != nil { logrus.Errorf("task %s getAllChainInfosMap err, %v", t.Name(), err) @@ -217,7 +220,7 @@ func (t *ChainInflowStatisticsTask) setStatisticsDataCache() { priceMap := cache.TokenPriceMap() for chain, _ := range chainInfosMap { - trendList, err := chainInflowStatisticsRepo.AggrTrend(chain, startTime, endTime) + trendList, err := chainInflowStatisticsRepo.AggrTrend(chain, allStartTime, endTime) if err != nil { logrus.Errorf("task %s AggrTrend %s err, %v", t.Name(), chain, err) continue @@ -225,16 +228,20 @@ func (t *ChainInflowStatisticsTask) setStatisticsDataCache() { volumeMap := make(map[string]decimal.Decimal, len(trendList)) totalDenomValue := decimal.Zero + allDaysTotalDenomValue := decimal.Zero for _, v := range trendList { denomAmount := decimal.NewFromFloat(v.DenomAmount) denomValue := ibctool.CalculateDenomValue(priceMap, v.BaseDenom, v.BaseDenomChain, denomAmount) - dt := time.Unix(v.SegmentStartTime, 0).Format(constant.DateFormat) - if vol, ok := volumeMap[dt]; ok { - volumeMap[dt] = vol.Add(denomValue) - } else { - volumeMap[dt] = denomValue + if v.SegmentStartTime >= startTime { + dt := time.Unix(v.SegmentStartTime, 0).Format(constant.DateFormat) + if vol, ok := volumeMap[dt]; ok { + volumeMap[dt] = vol.Add(denomValue) + } else { + volumeMap[dt] = denomValue + } + totalDenomValue = totalDenomValue.Add(denomValue) } - totalDenomValue = totalDenomValue.Add(denomValue) + allDaysTotalDenomValue = allDaysTotalDenomValue.Add(denomValue) } volumeItemList := make([]vo.VolumeItem, 0, len(volumeMap)) @@ -252,8 +259,13 @@ func (t *ChainInflowStatisticsTask) setStatisticsDataCache() { if err = chainFlowCacheRepo.SetInflowVolume(days, chain, totalDenomValue.String()); err != nil { logrus.Errorf("task %s SetInflowVolume %s err, %v", t.Name(), chain, err) } + + if err = chainFlowCacheRepo.SetInflowVolume(allDays, chain, allDaysTotalDenomValue.String()); err != nil { + logrus.Errorf("task %s SetInflowVolume all %s err, %v", t.Name(), chain, err) + } } chainFlowCacheRepo.ExpireInflowTrend(days, OneWeek*time.Second) chainFlowCacheRepo.ExpireInflowVolume(days, OneWeek*time.Second) + chainFlowCacheRepo.ExpireInflowVolume(allDays, OneWeek*time.Second) } diff --git a/internal/app/task/ibc_chain_outflow_statistics.go b/internal/app/task/ibc_chain_outflow_statistics.go index 3172cf7..553e555 100644 --- a/internal/app/task/ibc_chain_outflow_statistics.go +++ b/internal/app/task/ibc_chain_outflow_statistics.go @@ -208,6 +208,9 @@ func (t *ChainOutflowStatisticsTask) setStatisticsDataCache() { startTime, _ := lastNDaysZeroTimeUnix(days) _, endTime := todayUnix() + allDays := constant.ChainFlowVolumeDays + allStartTime, _ := lastNDaysZeroTimeUnix(allDays) + chainInfosMap, err := getAllChainInfosMap() if err != nil { logrus.Errorf("task %s getAllChainInfosMap err, %v", t.Name(), err) @@ -216,7 +219,7 @@ func (t *ChainOutflowStatisticsTask) setStatisticsDataCache() { priceMap := cache.TokenPriceMap() for chain, _ := range chainInfosMap { - trendList, err := chainOutflowStatisticsRepo.AggrTrend(chain, startTime, endTime) + trendList, err := chainOutflowStatisticsRepo.AggrTrend(chain, allStartTime, endTime) if err != nil { logrus.Errorf("task %s AggrTrend %s err, %v", t.Name(), chain, err) continue @@ -224,16 +227,20 @@ func (t *ChainOutflowStatisticsTask) setStatisticsDataCache() { volumeMap := make(map[string]decimal.Decimal, len(trendList)) totalDenomValue := decimal.Zero + allDaysTotalDenomValue := decimal.Zero for _, v := range trendList { denomAmount := decimal.NewFromFloat(v.DenomAmount) denomValue := ibctool.CalculateDenomValue(priceMap, v.BaseDenom, v.BaseDenomChain, denomAmount) - dt := time.Unix(v.SegmentStartTime, 0).Format(constant.DateFormat) - if vol, ok := volumeMap[dt]; ok { - volumeMap[dt] = vol.Add(denomValue) - } else { - volumeMap[dt] = denomValue + if v.SegmentStartTime >= startTime { + dt := time.Unix(v.SegmentStartTime, 0).Format(constant.DateFormat) + if vol, ok := volumeMap[dt]; ok { + volumeMap[dt] = vol.Add(denomValue) + } else { + volumeMap[dt] = denomValue + } + totalDenomValue = totalDenomValue.Add(denomValue) } - totalDenomValue = totalDenomValue.Add(denomValue) + allDaysTotalDenomValue = allDaysTotalDenomValue.Add(denomValue) } volumeItemList := make([]vo.VolumeItem, 0, len(volumeMap)) @@ -251,8 +258,13 @@ func (t *ChainOutflowStatisticsTask) setStatisticsDataCache() { if err = chainFlowCacheRepo.SetOutflowVolume(days, chain, totalDenomValue.String()); err != nil { logrus.Errorf("task %s SetOutflowVolume %s err, %v", t.Name(), chain, err) } + + if err = chainFlowCacheRepo.SetOutflowVolume(allDays, chain, allDaysTotalDenomValue.String()); err != nil { + logrus.Errorf("task %s SetOutflowVolume all %s err, %v", t.Name(), chain, err) + } } chainFlowCacheRepo.ExpireOutflowTrend(days, OneWeek*time.Second) chainFlowCacheRepo.ExpireOutflowVolume(days, OneWeek*time.Second) + chainFlowCacheRepo.ExpireOutflowVolume(allDays, OneWeek*time.Second) }