refactor(statistic): refactor statistic by comments

(lack enough test for queue and full-chain api)
1. refactor task queue with asynq serveMux and Scheduler
2. refactor statistic related api
3. get timezone from env(see timeutil.go)

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/duke-git/lancet/v2 v2.3.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.6 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -78,9 +79,9 @@ require (
 	github.com/vcaesar/cedar v0.20.2 // indirect
 	golang.org/x/arch v0.11.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
 	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sync v0.9.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.7.0 // indirect

go.sum

@@ -85,6 +85,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/duke-git/lancet/v2 v2.3.3 h1:OhqzNzkbJBS9ZlWLo/C7g+WSAOAAyNj7p9CAiEHurUc=
+github.com/duke-git/lancet/v2 v2.3.3/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -362,10 +364,13 @@ golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
 golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -378,6 +383,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -402,6 +409,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
 golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

handler/statistic.go

@@ -10,50 +10,32 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func GetAllStatisticHandler(c *gin.Context) {
-	statistics, err := service.GetAllStatistics(c)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误"})
-		return
-	}
-	c.JSON(http.StatusOK, dto.StatisticResponse{DailyInfos: statistics})
-}
-
-func GetDailyStatisticHandler(c *gin.Context) {
+func GetStatisticHandler(c *gin.Context) {
 	request := dto.StatisticRequest{
-		InDetail: false,
+		StartTime:  0,
+		EndTime:    0,
+		PeriodKeys: []model.PeriodInfoKey{},
 	}
 	if err := c.ShouldBindJSON(&request); err != nil {
 		c.JSON(http.StatusBadRequest, dto.BaseResponse{Message: "参数错误"})
 		return
 	}
-	if request.DateTime <= 0 {
+	if request.StartTime > request.EndTime {
 		c.JSON(http.StatusBadRequest, dto.BaseResponse{Message: "参数错误"})
 		return
 	}
-	if !request.InDetail {
-		minimal, err := service.GetDailyStatisticMinimal(c, time.Unix(request.DateTime, 0))
-		if err != nil {
-			c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误"})
-			return
-		}
-		detail := model.DailyInfoDetail{
-			DailyInfoMinimal: minimal,
-			WAU:              0,
-			MAU:              0,
-		}
-		c.JSON(http.StatusOK, dto.StatisticResponse{DailyInfos: []model.DailyInfoDetail{
-			detail,
-		}})
-		return
+	filter := model.StatisticFilter{
+		StartTime:      time.Unix(request.StartTime, 0),
+		EndTime:        time.Unix(request.EndTime, 0),
+		PeriodInfoKeys: request.PeriodKeys,
 	}
-
-	detail, err := service.GetDailyStatisticDetail(c, time.Unix(request.DateTime, 0))
+	dailyInfos, periodInfos, err := service.GetStatistics(c, filter)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误"})
 		return
 	}
-	c.JSON(http.StatusOK, dto.StatisticResponse{DailyInfos: []model.DailyInfoDetail{
-		detail,
-	}})
+	c.JSON(http.StatusOK, dto.StatisticResponse{
+		DailyInfos:  dailyInfos,
+		PeriodInfos: periodInfos,
+	})
 }

main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"jcourse_go/dal"
-	"jcourse_go/task"
 
 	"github.com/gin-gonic/gin"
 	"github.com/joho/godotenv"
@@ -12,7 +11,6 @@ func Init() {
 	_ = godotenv.Load()
 	dal.InitRedisClient()
 	dal.InitDBClient()
-	task.InitStatistic(dal.GetDBClient())
 }
 
 func main() {

middleware/statistic.go

@@ -5,6 +5,7 @@ import (
 	"jcourse_go/dal"
 	"jcourse_go/model/model"
 	"jcourse_go/repository"
+	"log"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -17,7 +18,7 @@ var UV = NewUVMiddleware()
 var PV = NewPVMiddleware()
 
 const (
-	DefaultDuplicateJudgeDuration = time.Second
+	DefaultDuplicateJudgeDuration = 10 * time.Minute
 	DuplicateJudgeDurationKey     = "duplicate_judge_duration"
 )
 
@@ -29,10 +30,24 @@ func UpdateDuplicateJudgeDuration(ctx context.Context) time.Duration {
 	siteQuery := repository.NewSettingQuery(dal.GetDBClient())
 	siteSetting, err := siteQuery.GetSetting(ctx, DuplicateJudgeDurationKey)
 	if err != nil {
+		log.Printf("failed to get site setting: %v", err)
 		LastQuerySiteSettingDuration = DefaultDuplicateJudgeDuration
 		return DefaultDuplicateJudgeDuration
 	}
-	return siteSetting.GetValue().(time.Duration)
+	// assert setting type is string
+	duration, err := time.ParseDuration(siteSetting.GetValue().(string))
+	if err != nil {
+		log.Printf("failed to parse setting: %v", siteSetting.GetValue())
+		LastQuerySiteSettingDuration = DefaultDuplicateJudgeDuration
+		return DefaultDuplicateJudgeDuration
+	}
+	if duration > 0 {
+		log.Printf("update duration to: %v", duration)
+		LastQuerySiteSettingDuration = duration
+		return duration
+	}
+	LastQuerySiteSettingDuration = DefaultDuplicateJudgeDuration
+	return DefaultDuplicateJudgeDuration
 }
 func (u *UserRequestCache) IsDuplicate(req string) bool {
 	u.mutex.RLock()

model/converter/statistic.go

@@ -1,28 +1,76 @@
 package converter
 
 import (
+	"fmt"
 	"jcourse_go/model/model"
 	"jcourse_go/model/po"
 	"jcourse_go/util"
 
+	"github.com/duke-git/lancet/v2/mathutil"
+
 	"github.com/RoaringBitmap/roaring"
 )
 
-func ConvertDailyInfoMinimalFromPO(po po.StatisticPO) model.DailyInfoMinimal {
-	return model.DailyInfoMinimal{
+func ConvertDailyInfoFromPO(po po.StatisticPO) model.DailyInfo {
+	return model.DailyInfo{
 		ID:               int64(po.ID),
 		Date:             util.FormatDate(po.Date),
 		UVCount:          po.UVCount,
 		PVCount:          po.PVCount,
-		NewCourseCount:   po.NewCourses,
 		NewUserCount:     po.NewUsers,
 		NewReviewCount:   po.NewReviews,
-		TotalCourseCount: po.TotalCourses,
 		TotalReviewCount: po.TotalReviews,
 		TotalUserCount:   po.TotalUsers,
 	}
 }
 
+// GetPeriodInfoFromPOs 从统计数据中获取指定的周期信息, 调用者保证pos中的数据是按时间增序排列的
+func GetPeriodInfoFromPOs(pos []po.StatisticPO, keys []model.PeriodInfoKey) (map[model.PeriodInfoKey][]model.PeriodInfo, error) {
+	const week = 7
+	const month = 30
+	periodInfoMap := make(map[model.PeriodInfoKey][]model.PeriodInfo)
+	for _, key := range keys {
+		switch key {
+		case model.PeriodInfoKeyMAU:
+			for i := range pos {
+				if i >= month-1 {
+					monthWindow := make([]int64, month)
+					for j := i - month + 1; j <= i; j++ {
+						monthWindow[j] = pos[j].UVCount
+					}
+					newInfo := model.PeriodInfo{
+						StartTime: pos[i-month+1].Date.Unix(),
+						EndTime:   pos[i].Date.Unix(),
+						Value:     mathutil.Average(monthWindow...),
+						Key:       key,
+					}
+					periodInfoMap[key] = append(periodInfoMap[key], newInfo)
+				}
+			}
+		case model.PeriodInfoKeyWAU:
+			periodInfoMap[key] = make([]model.PeriodInfo, 0)
+			for i := range pos {
+				if i >= week-1 {
+					weekWindow := make([]int64, week)
+					for j := i - week + 1; j <= i; j++ {
+						weekWindow[j] = pos[j].UVCount
+					}
+					newInfo := model.PeriodInfo{
+						StartTime: pos[i-week+1].Date.Unix(),
+						EndTime:   pos[i].Date.Unix(),
+						Value:     mathutil.Average(weekWindow...),
+						Key:       key,
+					}
+					periodInfoMap[key] = append(periodInfoMap[key], newInfo)
+				}
+			}
+		default:
+			return nil, fmt.Errorf(model.ErrInvalidPeriodInfoKey, key)
+		}
+	}
+	return periodInfoMap, nil
+}
+
 func ConvertUVDataFromPO(data []byte) (model.UVData, error) {
 	uv := roaring.New()
 	err := uv.UnmarshalBinary(data)

model/dto/statistic.go

@@ -3,10 +3,12 @@ package dto
 import "jcourse_go/model/model"
 
 type StatisticRequest struct {
-	DateTime int64 `json:"date_time" form:"date_time"` // unix timestamp, 单位秒, 如果不指定则默认为当天
-	InDetail bool  `json:"in_detail" form:"in_detail"`
+	StartTime  int64                 `json:"start_time" form:"start_time"` // unix timestamp, 单位秒
+	EndTime    int64                 `json:"end_time" form:"end_time"`     // unix timestamp, 单位秒
+	PeriodKeys []model.PeriodInfoKey `json:"period_keys" form:"period_keys"`
 }
 
 type StatisticResponse struct {
-	DailyInfos []model.DailyInfoDetail `json:"daily_infos"`
+	DailyInfos  []model.DailyInfo  `json:"daily_infos"`
+	PeriodInfos []model.PeriodInfo `json:"period_infos"`
 }

model/model/statistic.go

@@ -1,24 +1,45 @@
 package model
 
-import "github.com/RoaringBitmap/roaring"
+import (
+	"time"
 
-type DailyInfoMinimal struct {
+	"github.com/RoaringBitmap/roaring"
+)
+
+type DailyInfo struct {
 	ID               int64  `json:"id"`
 	Date             string `json:"date"` // yyyy-mm-dd
 	NewUserCount     int64  `json:"new_user_count"`
 	NewReviewCount   int64  `json:"new_review_count"`
-	NewCourseCount   int64  `json:"new_course_count"`
 	UVCount          int64  `json:"uv_count"`
 	PVCount          int64  `json:"pv_count"`
 	TotalUserCount   int64  `json:"total_user_count"`
 	TotalReviewCount int64  `json:"total_review_count"`
-	TotalCourseCount int64  `json:"total_course_count"`
+}
+type PeriodInfo struct {
+	StartTime int64  `json:"start_time"`
+	EndTime   int64  `json:"end_time"`
+	Key       string `json:"key"`
+	Value     int64  `json:"value"`
+}
+type PeriodInfoKey = string
+
+const ErrInvalidPeriodInfoKey = "invalid period info key: %s"
+
+const (
+	PeriodInfoKeyMAU PeriodInfoKey = "MAU"
+	PeriodInfoKeyWAU PeriodInfoKey = "WAU"
+)
+
+var PeriodInfoKeys = []PeriodInfoKey{
+	PeriodInfoKeyMAU,
+	PeriodInfoKeyWAU,
 }
 
-type DailyInfoDetail struct {
-	DailyInfoMinimal
-	WAU int64 `json:"wau"`
-	MAU int64 `json:"mau"`
+type StatisticFilter struct {
+	StartTime      time.Time
+	EndTime        time.Time
+	PeriodInfoKeys []PeriodInfoKey
 }
 
 type UVData = *roaring.Bitmap

model/po/statistic.go

@@ -11,12 +11,10 @@ type StatisticPO struct {
 	UVCount      int64
 	PVCount      int64
 	Date         time.Time `gorm:"index:,unique"` // createdAt time != actual datetime, 可能异步落盘, 统一为中午12点
-	NewCourses   int64
 	NewUsers     int64
 	NewReviews   int64
 	TotalUsers   int64 // 这几个是在db之中存储,按天更新,还是调用接口实时统计?
 	TotalReviews int64
-	TotalCourses int64
 }
 
 // StatisticDataPO uv bitmap, etc, separate date and count

repository/option.go

@@ -70,18 +70,22 @@ func WithDate(datetime time.Time) DBOption {
 }
 
 func WithDateBetween(start, end time.Time) DBOption {
+	startDate := util.GetMidTime(start)
+	endDate := util.GetMidTime(end)
 	return func(db *gorm.DB) *gorm.DB {
-		return db.Where("date between ? and ?", start, end)
+		return db.Where("date between ? and ?", startDate, endDate)
 	}
 }
 func WithDateAfter(start time.Time) DBOption {
+	startDate := util.GetMidTime(start)
 	return func(db *gorm.DB) *gorm.DB {
-		return db.Where("date >= ?", start)
+		return db.Where("date >= ?", startDate)
 	}
 }
 func WithDateBefore(end time.Time) DBOption {
+	endDate := util.GetMidTime(end)
 	return func(db *gorm.DB) *gorm.DB {
-		return db.Where("date <= ?", end)
+		return db.Where("date <= ?", endDate)
 	}
 }
 func WithDateOrder(ascending bool) DBOption {

router.go

@@ -76,8 +76,7 @@ func registerRouter(r *gin.Engine) {
 	userPointGroup.POST("/transfer", handler.TransferUserPointHandler)
 
 	statisticGroup := needAuthGroup.Group("/statistic")
-	statisticGroup.GET("", handler.GetAllStatisticHandler)
-	statisticGroup.GET("/daily", handler.GetDailyStatisticHandler)
+	statisticGroup.GET("", handler.GetStatisticHandler)
 
 	adminGroup := needAuthGroup.Group("/admin")
 	adminGroup.Use(middleware.RequireAdmin())