handle 429 error when calling mtm

v2/cmd/price_filler/main.go

@@ -27,8 +27,6 @@ import (
 	"go.uber.org/zap"
 )
 
-// This week I will deploy the new price filler of tradelog v2, which calls my new mark to market.
-// After deploying, I will have data to continue creating competition dashboard.
 func main() {
 	app := libapp.NewApp()
 	app.Name = "trade logs crawler service"
@@ -80,7 +78,7 @@ func run(c *cli.Context) error {
 		l.Errorw("Error while init price filler")
 		return err
 	}
-	priceFiller.Run(c.Int(libapp.FillPriceTimeIntervalFlag.Name))
+	priceFiller.Run(c.Duration(libapp.FillPriceTimeIntervalFlag.Name))
 	return nil
 }
 

v2/pkg/app/price_filler.go

@@ -1,6 +1,8 @@
 package app
 
 import (
+	"time"
+
 	"github.com/urfave/cli"
 )
 
@@ -19,9 +21,10 @@ var MarkToMarketURLFlag = cli.StringFlag{
 	EnvVar: "MARK_TO_MARKET_URL",
 }
 
-var FillPriceTimeIntervalFlag = cli.StringFlag{
+var FillPriceTimeIntervalFlag = cli.DurationFlag{
 	Name:   "fill-price-time-interval",
 	EnvVar: "FILL_PRICE_TIME_INTERVAL",
+	Value:  time.Minute,
 }
 
 func PriceFillerFlags() []cli.Flag {

v2/pkg/mtm/client.go

@@ -2,6 +2,7 @@ package mtm
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
@@ -10,6 +11,8 @@ import (
 	"github.com/KyberNetwork/tradinglib/pkg/httpclient"
 )
 
+var ErrRateLimit = errors.New("rate limit exceeded")
+
 type MtmClient struct {
 	baseURL    string
 	httpClient *http.Client
@@ -87,11 +90,16 @@ func (m *MtmClient) GetHistoricalRate(
 	}
 
 	var rate RateV3Response
-	if _, err := httpclient.DoHTTPRequest(
+	resp, err := httpclient.DoHTTPRequest(
 		m.httpClient,
 		httpReq,
 		&rate,
-		httpclient.WithStatusCode(http.StatusOK)); err != nil {
+		httpclient.WithStatusCode(http.StatusOK),
+	)
+	if resp.StatusCode == http.StatusTooManyRequests { // 429
+		return 0, ErrRateLimit
+	}
+	if err != nil {
 		return 0, fmt.Errorf("do http request error: %w", err)
 	}
 

v2/pkg/price_filler/price_filler.go

@@ -85,7 +85,7 @@ func NewPriceFiller(l *zap.SugaredLogger,
 	return p, nil
 }
 
-func (p *PriceFiller) Run(fillPriceInterval int) {
+func (p *PriceFiller) Run(fillPriceInterval time.Duration) {
 	go p.runUpdateAllCoinInfoRoutine()
 	p.runBackFillTradelogPriceRoutine(fillPriceInterval)
 }
@@ -140,8 +140,8 @@ func (p *PriceFiller) runUpdateAllCoinInfoRoutine() {
 	}
 }
 
-func (p *PriceFiller) runBackFillTradelogPriceRoutine(fillPriceInterval int) {
-	ticker := time.NewTicker(time.Duration(fillPriceInterval) * time.Minute)
+func (p *PriceFiller) runBackFillTradelogPriceRoutine(fillPriceInterval time.Duration) {
+	ticker := time.NewTicker(fillPriceInterval)
 	defer ticker.Stop()
 
 	for ; ; <-ticker.C {
@@ -199,19 +199,20 @@ func (p *PriceFiller) fullFillBebopTradeLog(tradeLog storageTypes.TradeLog) (sto
 	takerAmounts := strings.Split(tradeLog.TakerTokenAmount, ",")
 
 	makerPrice, makerSumUsdAmount, err := p.getSumAmountUsd(makerTokens, makerAmounts, int64(tradeLog.Timestamp))
-
-	if err != nil {
+	if isConnectionRefusedError(err) {
+		p.l.Errorw("Failed to getSumAndAmountUsd for maker", "err", err)
 		return tradeLog, err
 	}
 
-	tradeLog.MakerTokenPrice = &makerPrice // set zero price for multi-maker trade
-	tradeLog.MakerUsdAmount = &makerSumUsdAmount
-
 	takerPrice, takerUsdAmount, err := p.getSumAmountUsd(takerTokens, takerAmounts, int64(tradeLog.Timestamp))
-	if err != nil {
+	if isConnectionRefusedError(err) {
+		p.l.Errorw("Failed to getSumAmountUsd for taker", "err", err)
 		return tradeLog, err
 	}
 
+	tradeLog.MakerTokenPrice = &makerPrice // set zero price for multi-maker trade
+	tradeLog.MakerUsdAmount = &makerSumUsdAmount
+
 	tradeLog.TakerTokenPrice = &takerPrice
 	tradeLog.TakerUsdAmount = &takerUsdAmount
 
@@ -332,6 +333,9 @@ func isConnectionRefusedError(err error) bool {
 	if err == nil {
 		return false
 	}
+	if errors.Is(err, mtm.ErrRateLimit) {
+		return true
+	}
 	var netErr *net.OpError
 	if errors.As(err, &netErr) {
 		if strings.Contains(netErr.Err.Error(), "connection refused") {