From 3c5034620daf4b1707fecbcd517ca17213003626 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 17 Apr 2024 02:01:01 +0700 Subject: [PATCH] feat: api crud memos --- .../20240417112538-add_memo_logs_table.sql | 18 + pkg/handler/handler.go | 3 + pkg/handler/memologs/errs/errors.go | 8 + pkg/handler/memologs/interface.go | 9 + pkg/handler/memologs/memo_log.go | 323 ++++++++++++++++++ pkg/handler/memologs/request/request.go | 17 + pkg/model/memo_log.go | 52 +++ pkg/routes/v1.go | 7 + pkg/store/employee/employee.go | 14 + pkg/store/employee/interface.go | 1 + pkg/store/memolog/interface.go | 15 + pkg/store/memolog/memo_log.go | 32 ++ pkg/store/store.go | 3 + pkg/view/memo.go | 56 +++ 14 files changed, 558 insertions(+) create mode 100644 migrations/schemas/20240417112538-add_memo_logs_table.sql create mode 100644 pkg/handler/memologs/errs/errors.go create mode 100644 pkg/handler/memologs/interface.go create mode 100644 pkg/handler/memologs/memo_log.go create mode 100644 pkg/handler/memologs/request/request.go create mode 100644 pkg/model/memo_log.go create mode 100644 pkg/store/memolog/interface.go create mode 100644 pkg/store/memolog/memo_log.go create mode 100644 pkg/view/memo.go diff --git a/migrations/schemas/20240417112538-add_memo_logs_table.sql b/migrations/schemas/20240417112538-add_memo_logs_table.sql new file mode 100644 index 000000000..9498416a6 --- /dev/null +++ b/migrations/schemas/20240417112538-add_memo_logs_table.sql @@ -0,0 +1,18 @@ +-- +migrate Up +CREATE TABLE IF NOT EXISTS memo_logs ( + id UUID PRIMARY KEY DEFAULT (UUID()), + deleted_at TIMESTAMP(6), + created_at TIMESTAMP(6) DEFAULT (now()), + updated_at TIMESTAMP(6) DEFAULT (now()), + + title TEXT NOT NULL, + url TEXT NOT NULL, + authors JSONB, + tags JSONB, + description TEXT, + published_at TIMESTAMP(6) NOT NULL, + reward DECIMAL +); + +-- +migrate Down +DROP TABLE IF EXISTS memo_logs; diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 3976afffd..ee0946782 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -21,6 +21,7 @@ import ( "github.com/dwarvesf/fortress-api/pkg/handler/healthz" "github.com/dwarvesf/fortress-api/pkg/handler/icy" "github.com/dwarvesf/fortress-api/pkg/handler/invoice" + "github.com/dwarvesf/fortress-api/pkg/handler/memologs" "github.com/dwarvesf/fortress-api/pkg/handler/metadata" "github.com/dwarvesf/fortress-api/pkg/handler/notion" "github.com/dwarvesf/fortress-api/pkg/handler/payroll" @@ -53,6 +54,7 @@ type Handler struct { Feedback feedback.IHandler Healthcheck healthz.IHandler Invoice invoice.IHandler + MemoLog memologs.IHandler Metadata metadata.IHandler Notion notion.IHandler Payroll payroll.IHandler @@ -83,6 +85,7 @@ func New(store *store.Store, repo store.DBRepo, service *service.Service, ctrl * Feedback: feedback.New(store, repo, service, logger, cfg), Healthcheck: healthz.New(), Invoice: invoice.New(ctrl, store, repo, service, worker, logger, cfg), + MemoLog: memologs.New(ctrl, store, repo, service, logger, cfg), Metadata: metadata.New(store, repo, service, logger, cfg), Notion: notion.New(store, repo, service, logger, cfg), Payroll: payroll.New(ctrl, store, repo, service, worker, logger, cfg), diff --git a/pkg/handler/memologs/errs/errors.go b/pkg/handler/memologs/errs/errors.go new file mode 100644 index 000000000..0bf127a5b --- /dev/null +++ b/pkg/handler/memologs/errs/errors.go @@ -0,0 +1,8 @@ +package errs + +import "errors" + +var ( + ErrInvalidPublishedAt = errors.New("cannot parse publishedAt") + ErrInvalidDateFormat = errors.New("invalid date format") +) diff --git a/pkg/handler/memologs/interface.go b/pkg/handler/memologs/interface.go new file mode 100644 index 000000000..2e49fc2d4 --- /dev/null +++ b/pkg/handler/memologs/interface.go @@ -0,0 +1,9 @@ +package memologs + +import "github.com/gin-gonic/gin" + +type IHandler interface { + Create(c *gin.Context) + List(c *gin.Context) + Sync(c *gin.Context) +} diff --git a/pkg/handler/memologs/memo_log.go b/pkg/handler/memologs/memo_log.go new file mode 100644 index 000000000..869411977 --- /dev/null +++ b/pkg/handler/memologs/memo_log.go @@ -0,0 +1,323 @@ +package memologs + +import ( + "encoding/xml" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" + + "github.com/dwarvesf/fortress-api/pkg/config" + "github.com/dwarvesf/fortress-api/pkg/controller" + "github.com/dwarvesf/fortress-api/pkg/handler/memologs/request" + "github.com/dwarvesf/fortress-api/pkg/logger" + "github.com/dwarvesf/fortress-api/pkg/model" + "github.com/dwarvesf/fortress-api/pkg/service" + "github.com/dwarvesf/fortress-api/pkg/store" + "github.com/dwarvesf/fortress-api/pkg/view" + "github.com/gin-gonic/gin" + "github.com/shopspring/decimal" +) + +type handler struct { + controller *controller.Controller + store *store.Store + service *service.Service + logger logger.Logger + repo store.DBRepo + config *config.Config +} + +// New returns a handler +func New(controller *controller.Controller, store *store.Store, repo store.DBRepo, service *service.Service, logger logger.Logger, cfg *config.Config) IHandler { + return &handler{ + controller: controller, + store: store, + repo: repo, + service: service, + logger: logger, + config: cfg, + } +} + +func (h *handler) Create(c *gin.Context) { + l := h.logger.Fields( + logger.Fields{ + "handler": "memologs", + "method": "Create", + }, + ) + + body := request.CreateMemoLogsRequest{} + if err := c.ShouldBindJSON(&body); err != nil { + l.Error(err, "[memologs.Create] failed to decode body") + c.JSON(http.StatusBadRequest, view.CreateResponse[any](nil, nil, err, body, "")) + return + } + + memologs := make([]model.MemoLog, 0) + for _, b := range body { + publishedAt, _ := time.Parse(time.RFC3339Nano, b.PublishedAt) + authors := make([]model.MemoLogAuthor, 0) + for _, author := range b.Authors { + discordID, found := whitelistedUsers[author] + if !found { + discordMember, err := h.service.Discord.GetMemberByUsername(author) + if err != nil { + l.Errorf(err, "[memologs.Create] failed to get discord member", "author", author) + continue + } + discordID = discordMember.User.ID + } + + github := "" + employeeID := "" + + if discordID != "" { + empl, err := h.store.Employee.GetByDiscordID(h.repo.DB(), discordID, true) + if err != nil { + l.Errorf(err, "[brainerylogs.Create] failed to get employee with discord id %v", discordID) + } + + if empl != nil { + employeeID = empl.ID.String() + githubSA := model.SocialAccounts(empl.SocialAccounts).GetGithub() + if githubSA != nil { + github = githubSA.AccountID + } + + } + } + + authors = append(authors, model.MemoLogAuthor{ + DiscordID: discordID, + GithubID: github, + EmployeeID: employeeID, + }) + } + + b := model.MemoLog{ + Title: b.Title, + URL: b.URL, + Authors: authors, + Tags: b.Tags, + PublishedAt: &publishedAt, + Description: b.Description, + Reward: b.Reward, + } + memologs = append(memologs, b) + } + + logs, err := h.store.MemoLog.Create(h.repo.DB(), memologs) + if err != nil { + l.Errorf(err, "[memologs.Create] failed to create new memo logs", "memologs", memologs) + c.JSON(http.StatusInternalServerError, view.CreateResponse[any](nil, nil, err, memologs, "")) + return + } + + c.JSON(http.StatusOK, view.CreateResponse[any](view.ToMemoLog(logs), nil, nil, body, "")) +} + +func (h *handler) List(c *gin.Context) { + l := h.logger.Fields( + logger.Fields{ + "handler": "memologs", + "method": "List", + }, + ) + + memoLogs, err := h.store.MemoLog.List(h.repo.DB()) + if err != nil { + l.Error(err, "failed to get memologs") + c.JSON(http.StatusInternalServerError, view.CreateResponse[any](nil, nil, err, nil, "")) + return + } + + c.JSON(http.StatusOK, view.CreateResponse[any](view.ToMemoLog(memoLogs), nil, nil, nil, "")) +} + +func (h *handler) Sync(c *gin.Context) { + l := h.logger.Fields( + logger.Fields{ + "handler": "memologs", + "method": "Sync", + }, + ) + + list, err := h.store.MemoLog.List(h.repo.DB()) + if err != nil { + l.Error(err, "[memologs.Sync] failed to get memo logs") + c.JSON(http.StatusInternalServerError, view.CreateResponse[any](nil, nil, err, nil, "")) + return + + } + + mapMemo := make(map[string]bool) + for _, memo := range list { + mapMemo[memo.URL] = true + } + + url := "https://memo.d.foundation/index.xml" + response, err := http.Get(url) + + if err != nil { + log.Fatal(err) + } + + data, err := io.ReadAll(response.Body) + if err != nil { + log.Fatal(err) + } + + d := Rss{} + err = xml.Unmarshal(data, &d) + + if err != nil { + log.Fatal(err) + } + + authorsMap := make(map[string]model.MemoLogAuthor) + memoLogs := make([]model.MemoLog, 0) + for _, item := range d.Channel.Item { + if mapMemo[item.Link] { + continue + } + + authorStrList := strings.Split(item.Author, ",") + memoAuthors := make([]model.MemoLogAuthor, 0) + for _, author := range authorStrList { + author = strings.TrimSpace(author) + if author == "" { + continue + } + + validatedAuthor, found := whitelistedUsers[author] + if !found { + continue + } + + dt, found := authorsMap[validatedAuthor] + if found { + memoAuthors = append(memoAuthors, dt) + continue + } + + empl, err := h.store.Employee.GetByDiscordUsername(h.repo.DB(), validatedAuthor) + if err != nil { + l.Error(err, fmt.Sprintf("[memologs.Sync] failed to get employee with discord username %v", validatedAuthor)) + } + + github := "" + discord := validatedAuthor + employeeID := "" + if empl != nil { + employeeID = empl.ID.String() + githubSA := model.SocialAccounts(empl.SocialAccounts).GetGithub() + if githubSA != nil { + github = githubSA.AccountID + } + + if empl.DiscordAccount != nil { + discord = empl.DiscordAccount.DiscordID + } + + } + + if github != "" || discord != "" || employeeID != "" { + memoAuthor := model.MemoLogAuthor{ + GithubID: github, + DiscordID: discord, + EmployeeID: employeeID, + } + authorsMap[validatedAuthor] = memoAuthor + memoAuthors = append(memoAuthors, memoAuthor) + } + } + + layout := "Mon, 02 Jan 2006 15:04:05 -0700" + publishedAt, err := time.Parse(layout, item.PubDate) + if err != nil { + l.Error(err, fmt.Sprintf("[memologs.Sync] failed to parse date %v", item.PubDate)) + } + + memoLogs = append(memoLogs, model.MemoLog{ + Title: item.Title, + URL: item.Link, + Authors: memoAuthors, + Description: item.Description, + PublishedAt: &publishedAt, + Reward: decimal.Decimal{}, + }) + } + + if len(memoLogs) == 0 { + c.JSON(http.StatusOK, view.CreateResponse[any](nil, nil, nil, nil, "no new memo logs")) + return + } + + results, err := h.store.MemoLog.Create(h.repo.DB(), memoLogs) + if err != nil { + l.Error(err, "[memologs.Sync] failed to create memo logs") + c.JSON(http.StatusInternalServerError, view.CreateResponse[any](nil, nil, err, nil, "")) + return + + } + + c.JSON(http.StatusOK, view.CreateResponse[any](view.ToMemoLog(results), nil, nil, nil, "ok")) +} + +var whitelistedUsers = map[string]string{ + "thanh": "790170208228212766", + "giangthan": "797051426437595136", + "nikki": "796991130184187944", + "anna": "575525326181105674", + "monotykamary": "184354519726030850", + "vhbien": "421992793582469130", + "minhcloud": "1007496699511570503", + "mickwan1234": "383793994271948803", + "hnh": "567326528216760320", + "duy": "788351441097195520", + "huytq": "361172853326086144", + "han": "151497832853929986", + "namtran": "785756392363524106", + "innno_": "753995829559165044", + "dudaka": "282500790692741121", + "tom": "184354519726030850", +} + +type Rss struct { + XMLName xml.Name `xml:"rss"` + Text string `xml:",chardata"` + Atom string `xml:"atom,attr"` + Version string `xml:"version,attr"` + Script string `xml:"script"` + Channel struct { + Text string `xml:",chardata"` + Title string `xml:"title"` + Link struct { + Text string `xml:",chardata"` + Href string `xml:"href,attr"` + Rel string `xml:"rel,attr"` + Type string `xml:"type,attr"` + } `xml:"link"` + Description string `xml:"description"` + Generator string `xml:"generator"` + Language string `xml:"language"` + ManagingEditor string `xml:"managingEditor"` + WebMaster string `xml:"webMaster"` + Copyright string `xml:"copyright"` + LastBuildDate string `xml:"lastBuildDate"` + Item []struct { + Text string `xml:",chardata"` + Title string `xml:"title"` + Link string `xml:"link"` + PubDate string `xml:"pubDate"` + Author string `xml:"author"` + Guid string `xml:"guid"` + Description string `xml:"description"` + Draft string `xml:"draft"` + } `xml:"item"` + } `xml:"channel"` +} diff --git a/pkg/handler/memologs/request/request.go b/pkg/handler/memologs/request/request.go new file mode 100644 index 000000000..24daff068 --- /dev/null +++ b/pkg/handler/memologs/request/request.go @@ -0,0 +1,17 @@ +package request + +import ( + "github.com/shopspring/decimal" +) + +type CreateMemoLogsRequest []MemoLogItem // @name CreateMemoLogsRequest + +type MemoLogItem struct { + Title string `json:"title" binding:"required"` + URL string `json:"url" binding:"required"` + Authors []string `json:"authors"` + Tags []string `json:"tags"` + Description string `json:"description"` + PublishedAt string `json:"publishedAt"` + Reward decimal.Decimal `json:"reward"` +} diff --git a/pkg/model/memo_log.go b/pkg/model/memo_log.go new file mode 100644 index 000000000..4ed5123cf --- /dev/null +++ b/pkg/model/memo_log.go @@ -0,0 +1,52 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" + "time" + + "github.com/shopspring/decimal" +) + +type MemoLog struct { + BaseModel + + Title string + URL string + Authors MemoLogAuthors + Tags JSONArrayString + Description string + PublishedAt *time.Time + Reward decimal.Decimal +} + +type MemoLogAuthors []MemoLogAuthor + +// MemoLogAuthor is the author of the memo log +type MemoLogAuthor struct { + EmployeeID string `json:"employeeID"` + GithubID string `json:"githubID"` + DiscordID string `json:"discordID"` +} + +func (j MemoLogAuthors) Value() (driver.Value, error) { + return json.Marshal(j) +} + +func (j *MemoLogAuthors) Scan(value interface{}) error { + if value == nil { + *j = nil + return nil + } + switch t := value.(type) { + case []uint8: + jsonData := value.([]uint8) + if string(jsonData) == "null" { + return nil + } + return json.Unmarshal(jsonData, j) + default: + return fmt.Errorf("could not scan type %T into json", t) + } +} diff --git a/pkg/routes/v1.go b/pkg/routes/v1.go index 61ceb6f24..7963beeca 100644 --- a/pkg/routes/v1.go +++ b/pkg/routes/v1.go @@ -334,6 +334,13 @@ func loadV1Routes(r *gin.Engine, h *handler.Handler, repo store.DBRepo, s *store braineryGroup.POST("/sync", amw.WithAuth, pmw.WithPerm(model.PermissionCronjobExecute), h.BraineryLog.Sync) } + memoGroup := v1.Group("/memos") + { + memoGroup.POST("", amw.WithAuth, h.MemoLog.Create) + memoGroup.POST("/sync", amw.WithAuth, h.MemoLog.Sync) + memoGroup.GET("", amw.WithAuth, h.MemoLog.List) + } + // Delivery metrics { deliveryGroup := v1.Group("/delivery-metrics") diff --git a/pkg/store/employee/employee.go b/pkg/store/employee/employee.go index ae2f559ed..cb8a42583 100644 --- a/pkg/store/employee/employee.go +++ b/pkg/store/employee/employee.go @@ -262,6 +262,20 @@ func (s *store) GetByDiscordID(db *gorm.DB, discordID string, preload bool) (*mo return employee, query.First(&employee).Error } +func (s *store) GetByDiscordUsername(db *gorm.DB, discordUsername string) (*model.Employee, error) { + var employee *model.Employee + query := db.Joins("JOIN discord_accounts ON discord_accounts.id = employees.discord_account_id AND discord_accounts.username = ?", discordUsername). + Where("employees.deleted_at IS NULL AND employees.working_status <> ?", model.WorkingStatusLeft). + Preload("SocialAccounts", "deleted_at IS NULL"). + Preload("DiscordAccount", "deleted_at IS NULL") + err := query.First(&employee).Error + if err != nil { + return nil, err + + } + return employee, nil +} + func (s *store) ListByDiscordRequest(db *gorm.DB, in DiscordRequestFilter, preload bool) ([]model.Employee, error) { var employee []model.Employee query := db.Where("employees.deleted_at IS NULL AND employees.working_status <> ?", model.WorkingStatusLeft).Order("employees.joined_date DESC") diff --git a/pkg/store/employee/interface.go b/pkg/store/employee/interface.go index 2b21f2f0a..ea94f8686 100644 --- a/pkg/store/employee/interface.go +++ b/pkg/store/employee/interface.go @@ -22,6 +22,7 @@ type IStore interface { GetLineManagersOfPeers(db *gorm.DB, employeeID string) ([]*model.Employee, error) GetMenteesByID(db *gorm.DB, employeeID string) ([]*model.Employee, error) GetByDiscordID(db *gorm.DB, discordID string, preload bool) (*model.Employee, error) + GetByDiscordUsername(db *gorm.DB, discordUsername string) (*model.Employee, error) ListByDiscordRequest(db *gorm.DB, in DiscordRequestFilter, preload bool) ([]model.Employee, error) ListWithMMAScore(db *gorm.DB) ([]model.EmployeeMMAScoreData, error) SimpleList(db *gorm.DB) ([]*model.Employee, error) diff --git a/pkg/store/memolog/interface.go b/pkg/store/memolog/interface.go new file mode 100644 index 000000000..48318355b --- /dev/null +++ b/pkg/store/memolog/interface.go @@ -0,0 +1,15 @@ +package memolog + +import ( + "time" + + "gorm.io/gorm" + + "github.com/dwarvesf/fortress-api/pkg/model" +) + +type IStore interface { + Create(db *gorm.DB, b []model.MemoLog) ([]model.MemoLog, error) + GetLimitByTimeRange(db *gorm.DB, start, end *time.Time, limit int) ([]model.MemoLog, error) + List(db *gorm.DB) ([]model.MemoLog, error) +} diff --git a/pkg/store/memolog/memo_log.go b/pkg/store/memolog/memo_log.go new file mode 100644 index 000000000..99468d357 --- /dev/null +++ b/pkg/store/memolog/memo_log.go @@ -0,0 +1,32 @@ +package memolog + +import ( + "time" + + "gorm.io/gorm" + + "github.com/dwarvesf/fortress-api/pkg/model" +) + +type store struct{} + +func New() IStore { + return &store{} +} + +// Create creates a memo log record in the database +func (s *store) Create(db *gorm.DB, b []model.MemoLog) ([]model.MemoLog, error) { + return b, db.Create(b).Error +} + +// GetLimitByTimeRange gets memo logs in a specific time range, with limit +func (s *store) GetLimitByTimeRange(db *gorm.DB, start, end *time.Time, limit int) ([]model.MemoLog, error) { + var logs []model.MemoLog + return logs, db.Where("published_at BETWEEN ? AND ?", start, end).Limit(limit).Order("published_at DESC").Find(&logs).Error +} + +// List gets all memo logs +func (s *store) List(db *gorm.DB) ([]model.MemoLog, error) { + var logs []model.MemoLog + return logs, db.Order("published_at DESC").Find(&logs).Error +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 003fb688b..2aabe6030 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -49,6 +49,7 @@ import ( "github.com/dwarvesf/fortress-api/pkg/store/icytransaction" "github.com/dwarvesf/fortress-api/pkg/store/invoice" "github.com/dwarvesf/fortress-api/pkg/store/invoicenumbercaching" + "github.com/dwarvesf/fortress-api/pkg/store/memolog" "github.com/dwarvesf/fortress-api/pkg/store/onleaverequest" "github.com/dwarvesf/fortress-api/pkg/store/operationalservice" "github.com/dwarvesf/fortress-api/pkg/store/organization" @@ -126,6 +127,7 @@ type Store struct { IcyTransaction icytransaction.IStore Invoice invoice.IStore InvoiceNumberCaching invoicenumbercaching.IStore + MemoLog memolog.IStore MonthlyDeliveryMetric deliverymetricmonthly.IStore OnLeaveRequest onleaverequest.IStore OperationalService operationalservice.IStore @@ -205,6 +207,7 @@ func New() *Store { IcyTransaction: icytransaction.New(), Invoice: invoice.New(), InvoiceNumberCaching: invoicenumbercaching.New(), + MemoLog: memolog.New(), MonthlyDeliveryMetric: deliverymetricmonthly.New(), OnLeaveRequest: onleaverequest.New(), OperationalService: operationalservice.New(), diff --git a/pkg/view/memo.go b/pkg/view/memo.go new file mode 100644 index 000000000..f54b3ef1c --- /dev/null +++ b/pkg/view/memo.go @@ -0,0 +1,56 @@ +package view + +import ( + "time" + + "github.com/dwarvesf/fortress-api/pkg/model" + "github.com/shopspring/decimal" +) + +type MemoLog struct { + ID string `json:"id"` + Title string `json:"title"` + URL string `json:"url"` + Authors []MemoLogAuthor `json:"authors"` + Description string `json:"description"` + PublishedAt *time.Time `json:"publishedAt"` + Reward decimal.Decimal `json:"reward"` +} // @name MemoLog + +// MemoLogAuthor is the author of the memo log +type MemoLogAuthor struct { + EmployeeID string `json:"employeeID"` + GithubID string `json:"githubID"` + DiscordID string `json:"discordID"` +} + +// MemoLogsResponse response for memo logs +type MemoLogsResponse struct { + Data []MemoLog `json:"data"` +} // @name MemoLogsResponse + +func ToMemoLog(memoLogs []model.MemoLog) []MemoLog { + rs := make([]MemoLog, 0) + for _, memoLog := range memoLogs { + authors := make([]MemoLogAuthor, 0) + for _, author := range memoLog.Authors { + authors = append(authors, MemoLogAuthor{ + EmployeeID: author.EmployeeID, + GithubID: author.GithubID, + DiscordID: author.DiscordID, + }) + } + + rs = append(rs, MemoLog{ + ID: memoLog.ID.String(), + Title: memoLog.Title, + URL: memoLog.URL, + Authors: authors, + Description: memoLog.Description, + PublishedAt: memoLog.PublishedAt, + Reward: memoLog.Reward, + }) + } + + return rs +}