diff --git a/go.mod b/go.mod index d42f670..83ed460 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/bytedance/sonic v1.12.1 github.com/gin-gonic/contrib v0.0.0-20240508051311-c1c6bf0061b0 github.com/gin-gonic/gin v1.10.0 + github.com/go-ego/gse v0.80.3 github.com/go-redis/redismock/v9 v9.2.0 github.com/gorilla/csrf v1.7.2 github.com/gwatts/gin-adapter v1.0.0 @@ -63,6 +64,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/vcaesar/cedar v0.20.2 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/net v0.25.0 // indirect diff --git a/go.sum b/go.sum index c300f68..5297578 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/gin-gonic/contrib v0.0.0-20240508051311-c1c6bf0061b0 h1:EUFmvQ8ffefnS github.com/gin-gonic/contrib v0.0.0-20240508051311-c1c6bf0061b0/go.mod h1:iqneQ2Df3omzIVTkIfn7c1acsVnMGiSLn4XF5Blh3Yg= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ego/gse v0.80.3 h1:YNFkjMhlhQnUeuoFcUEd1ivh6SOB764rT8GDsEbDiEg= +github.com/go-ego/gse v0.80.3/go.mod h1:Gt3A9Ry1Eso2Kza4MRaiZ7f2DTAvActmETY46Lxg0gU= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -164,6 +166,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/vcaesar/cedar v0.20.2 h1:TDx7AdZhilKcfE1WvdToTJf5VrC/FXcUOW+KY1upLZ4= +github.com/vcaesar/cedar v0.20.2/go.mod h1:lyuGvALuZZDPNXwpzv/9LyxW+8Y6faN7zauFezNsnik= +github.com/vcaesar/tt v0.20.1 h1:D/jUeeVCNbq3ad8M7hhtB3J9x5RZ6I1n1eZ0BJp7M+4= +github.com/vcaesar/tt v0.20.1/go.mod h1:cH2+AwGAJm19Wa6xvEa+0r+sXDJBT0QgNQey6mwqLeU= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= diff --git a/model/domain/course.go b/model/domain/course.go index d283e6d..ab40196 100644 --- a/model/domain/course.go +++ b/model/domain/course.go @@ -65,6 +65,7 @@ type TrainingPlanFilter struct { Department string EntryYear string ContainCourseIDs []int64 + SearchQuery string } type BaseCourse struct { @@ -80,4 +81,5 @@ type CourseListFilter struct { Departments []string Categories []string Credits []float64 + SearchQuery string } diff --git a/model/domain/review.go b/model/domain/review.go index dfa7822..4a8907c 100644 --- a/model/domain/review.go +++ b/model/domain/review.go @@ -3,12 +3,13 @@ package domain import "time" type ReviewFilter struct { - Page int64 - PageSize int64 - CourseID int64 - Semester string - UserID int64 - ReviewID int64 + Page int64 + PageSize int64 + CourseID int64 + Semester string + UserID int64 + ReviewID int64 + SearchQuery string } type Review struct { diff --git a/model/domain/teacher.go b/model/domain/teacher.go index 41b11b7..d7591c1 100644 --- a/model/domain/teacher.go +++ b/model/domain/teacher.go @@ -24,4 +24,5 @@ type TeacherListFilter struct { Pinyin string PinyinAbbr string ContainCourseIDs []int64 + SearchQuery string } diff --git a/model/domain/user.go b/model/domain/user.go index 102c2ed..b97423b 100644 --- a/model/domain/user.go +++ b/model/domain/user.go @@ -39,7 +39,8 @@ type UserProfile struct { } type UserFilter struct { - Page int64 - PageSize int64 + Page int64 + PageSize int64 + SearchQuery string // To be continued ... (add more fields) } diff --git a/model/dto/course.go b/model/dto/course.go index 104b927..31b0768 100644 --- a/model/dto/course.go +++ b/model/dto/course.go @@ -48,6 +48,7 @@ type CourseListRequest struct { Departments string `json:"departments" form:"departments"` Categories string `json:"categories" form:"categories"` Credits string `json:"credits" form:"credits"` + SearchQuery string `json:"search_query" form:"search_query"` } type CourseListResponse = BasePaginateResponse[CourseListItemDTO] diff --git a/model/dto/review.go b/model/dto/review.go index 83e5cd0..e84d04b 100644 --- a/model/dto/review.go +++ b/model/dto/review.go @@ -34,8 +34,9 @@ type CreateReviewResponse struct { } type ReviewListRequest struct { - Page int64 `json:"page" form:"page"` - PageSize int64 `json:"page_size" form:"page_size"` + Page int64 `json:"page" form:"page"` + PageSize int64 `json:"page_size" form:"page_size"` + SearchQuery string `json:"search_query" form:"search_query"` } type ReviewListResponse = BasePaginateResponse[ReviewDTO] diff --git a/model/dto/teacher.go b/model/dto/teacher.go index eb199fc..45e8a3e 100644 --- a/model/dto/teacher.go +++ b/model/dto/teacher.go @@ -18,14 +18,15 @@ type TeacherDetailRequest struct { } type TeacherListRequest struct { - Page int64 `json:"page" form:"page"` - PageSize int64 `json:"page_size" form:"page_size"` - Name string `json:"name" form:"name"` - Code string `json:"code" form:"code"` - Department string `json:"departments" form:"departments"` - Title string `json:"title" form:"title"` - Pinyin string `json:"pinyin" form:"pinyin"` - PinyinAbbr string `json:"pinyin_abbr" form:"pinyin_abbr"` + Page int64 `json:"page" form:"page"` + PageSize int64 `json:"page_size" form:"page_size"` + Name string `json:"name" form:"name"` + Code string `json:"code" form:"code"` + Department string `json:"departments" form:"departments"` + Title string `json:"title" form:"title"` + Pinyin string `json:"pinyin" form:"pinyin"` + PinyinAbbr string `json:"pinyin_abbr" form:"pinyin_abbr"` + SearchQuery string `json:"search_query" form:"search_query"` } type TeacherListResponse = BasePaginateResponse[TeacherDTO] diff --git a/model/dto/trainingplan.go b/model/dto/trainingplan.go index 92b221a..c075270 100644 --- a/model/dto/trainingplan.go +++ b/model/dto/trainingplan.go @@ -37,6 +37,7 @@ type TrainingPlanListQueryRequest struct { SortBy string `json:"sort_by" form:"sort_by"` Page int `json:"page" binding:"required" form:"page"` PageSize int `json:"page_size" binding:"required" form:"page_size"` + SearchQuery string `json:"search_query" form:"search_query"` } type TrainingPlanListRequest struct { Page int `json:"page" binding:"required" form:"page"` diff --git a/model/dto/user.go b/model/dto/user.go index 257b704..e6a3737 100644 --- a/model/dto/user.go +++ b/model/dto/user.go @@ -5,8 +5,9 @@ type UserRole = string type UserType = string type UserListRequest struct { - Page int64 `json:"page" form:"page"` - PageSize int64 `json:"page_size" form:"page_size"` + Page int64 `json:"page" form:"page"` + PageSize int64 `json:"page_size" form:"page_size"` + SearchQuery string `json:"search_query" form:"search_query"` } type UserListResponse = BasePaginateResponse[UserDetailDTO] diff --git a/model/po/course.go b/model/po/course.go index 57e23fa..9314d4a 100644 --- a/model/po/course.go +++ b/model/po/course.go @@ -21,6 +21,8 @@ type CoursePO struct { MainTeacherID int64 `gorm:"index;index:uniq_course,unique"` MainTeacherName string `gorm:"index"` Department string `gorm:"index;index:uniq_course,unique"` + + SearchIndex SearchIndex `gorm:"->:false;<-"` } func (po *CoursePO) TableName() string { diff --git a/model/po/review.go b/model/po/review.go index 38e375f..9a1e130 100644 --- a/model/po/review.go +++ b/model/po/review.go @@ -10,6 +10,7 @@ type ReviewPO struct { Rating int64 `gorm:"index"` Semester string `gorm:"index;index:uniq_course_review,unique"` IsAnonymous bool + SearchIndex SearchIndex `gorm:"->:false;<-"` } func (po *ReviewPO) TableName() string { diff --git a/model/po/search.go b/model/po/search.go new file mode 100755 index 0000000..247ec77 --- /dev/null +++ b/model/po/search.go @@ -0,0 +1,93 @@ +package po + +import ( + "context" + "database/sql/driver" + "jcourse_go/util" + "strings" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type SearchIndex string + +// ref: https://gorm.io/zh_CN/docs/data_types.html + +func (i *SearchIndex) Scan(value interface{}) error { return nil } +func (i SearchIndex) Value() (driver.Value, error) { return nil, nil } +func (i SearchIndex) GormDataType() string { return "tsvector" } + +func (i SearchIndex) GormValue(ctx context.Context, db *gorm.DB) clause.Expr { + return clause.Expr{ + SQL: "to_tsvector('simple', ?)", + Vars: []interface{}{string(i)}, + } +} + +func toIndex(fields []string) SearchIndex { + var sb strings.Builder + for _, field := range fields { + for _, segment := range util.SegWord(field) { + sb.WriteString(segment) + sb.WriteByte(' ') + } + } + return SearchIndex(sb.String()) +} + +// ref: https://gorm.io/zh_CN/docs/hooks.html + +func (c *CoursePO) BeforeCreate(*gorm.DB) error { + if c.SearchIndex == "" { + c.SearchIndex = toIndex([]string{ + c.Name, + c.Code, // 前缀模糊匹配更为适合 + c.MainTeacherName, + c.Department, // 不分词更为适合 + }) + } + return nil +} +func (c *CoursePO) BeforeSave(tx *gorm.DB) error { + return c.BeforeCreate(tx) +} + +func (t *TeacherPO) BeforeCreate(*gorm.DB) error { + if t.SearchIndex == "" { + t.SearchIndex = toIndex([]string{ + t.Name, + t.Department, + t.Code, + }) + } + return nil +} +func (t *TeacherPO) BeforeSave(tx *gorm.DB) error { + return t.BeforeCreate(tx) +} + +func (t *TrainingPlanPO) BeforeCreate(*gorm.DB) error { + if t.SearchIndex == "" { + t.SearchIndex = toIndex([]string{ + t.Major, + t.Department, + }) + } + return nil +} +func (t *TrainingPlanPO) BeforeSave(tx *gorm.DB) error { + return t.BeforeCreate(tx) +} + +func (r *ReviewPO) BeforeCreate(*gorm.DB) error { + if r.SearchIndex == "" { + r.SearchIndex = toIndex([]string{ + r.Comment, + }) + } + return nil +} +func (r *ReviewPO) BeforeSave(tx *gorm.DB) error { + return r.BeforeCreate(tx) +} diff --git a/model/po/teacher.go b/model/po/teacher.go index 11a5a5d..2faa4f7 100644 --- a/model/po/teacher.go +++ b/model/po/teacher.go @@ -14,6 +14,8 @@ type TeacherPO struct { Picture string // picture URL ProfileURL string Biography string // 个人简述 + + SearchIndex SearchIndex `gorm:"->:false;<-"` } func (po *TeacherPO) TableName() string { diff --git a/model/po/trainingplan.go b/model/po/trainingplan.go index a7d2ab3..1262649 100644 --- a/model/po/trainingplan.go +++ b/model/po/trainingplan.go @@ -12,6 +12,8 @@ type TrainingPlanPO struct { TotalYear int `gorm:"index;index:uniq_training_plan,unique"` MinCredits float64 `gorm:"index;index:uniq_training_plan,unique"` MajorClass string `gorm:"index;index:uniq_training_plan,unique"` // the class of major + + SearchIndex SearchIndex `gorm:"->:false;<-"` } func (po *TrainingPlanPO) TableName() string { diff --git a/repository/course.go b/repository/course.go index cb2df60..19077bb 100644 --- a/repository/course.go +++ b/repository/course.go @@ -103,6 +103,7 @@ type ICourseQuery interface { WithMainTeacherID(id int64) DBOption WithLimit(limit int64) DBOption WithOffset(offset int64) DBOption + WithSearch(query string) DBOption } type CourseQuery struct { diff --git a/repository/review.go b/repository/review.go index 4047ffb..202ab82 100644 --- a/repository/review.go +++ b/repository/review.go @@ -24,6 +24,7 @@ type IReviewQuery interface { WithOrderBy(orderBy string, ascending bool) DBOption WithLimit(limit int64) DBOption WithOffset(offset int64) DBOption + WithSearch(query string) DBOption } type ReviewQuery struct { diff --git a/repository/search.go b/repository/search.go new file mode 100755 index 0000000..6762258 --- /dev/null +++ b/repository/search.go @@ -0,0 +1,49 @@ +package repository + +import ( + "jcourse_go/util" + "strings" + + "gorm.io/gorm" +) + +func (*CourseQuery) WithSearch(query string) DBOption { return withSearch(query) } +func (*ReviewQuery) WithSearch(query string) DBOption { return withSearch(query) } +func (*TeacherQuery) WithSearch(query string) DBOption { return withSearch(query) } +func (*TrainingPlanQuery) WithSearch(query string) DBOption { return withSearch(query) } + +func withSearch(query string) DBOption { + return func(db *gorm.DB) *gorm.DB { + return db.Where("search_index @@ to_tsquery('simple', ?)", + userQueryToTsQuery(query), + ) + } +} + +// 目前只搜用户名 +func (*UserQuery) WithSearch(query string) DBOption { + return func(db *gorm.DB) *gorm.DB { + return db.Where("name LIKE ?", query+"%") + } +} + +// 空格分割的每个词都要匹配，词内分词做模糊匹配 +func userQueryToTsQuery(query string) string { + var sb strings.Builder + words := strings.Fields(query) + for i, word := range words { + if i != 0 { + sb.WriteString(" & ") + } + sb.WriteByte('(') + segs := util.SegWord(word) + for j, seg := range segs { + if j != 0 { + sb.WriteString(" | ") + } + sb.WriteString(seg) + } + sb.WriteByte(')') + } + return sb.String() +} diff --git a/repository/teacher.go b/repository/teacher.go index 1da17be..13351f4 100644 --- a/repository/teacher.go +++ b/repository/teacher.go @@ -26,6 +26,7 @@ type ITeacherQuery interface { WithProfileURL(profileURL string) DBOption WithIDs(ids []int64) DBOption WithPaginate(page int64, pageSize int64) DBOption + WithSearch(query string) DBOption } type TeacherQuery struct { diff --git a/repository/trainingplan.go b/repository/trainingplan.go index d74481d..d3d6aa8 100644 --- a/repository/trainingplan.go +++ b/repository/trainingplan.go @@ -25,6 +25,7 @@ type ITrainingPlanQuery interface { WithDegree(degree string) DBOption WithIDs(courseIDs []int64) DBOption WithPaginate(page int64, pageSize int64) DBOption + WithSearch(query string) DBOption } func NewTrainingPlanQuery() ITrainingPlanQuery { diff --git a/repository/user.go b/repository/user.go index aa9caee..bc5d435 100644 --- a/repository/user.go +++ b/repository/user.go @@ -28,6 +28,7 @@ type IUserQuery interface { ResetUserPassword(ctx context.Context, userID int64, password string) error WithLimit(limit int64) DBOption WithOffset(offset int64) DBOption + WithSearch(query string) DBOption } type IUserProfileQuery interface { diff --git a/service/course.go b/service/course.go index 18cf01a..a1a1857 100644 --- a/service/course.go +++ b/service/course.go @@ -68,6 +68,9 @@ func buildCourseDBOptionFromFilter(query repository.ICourseQuery, filter domain. if len(filter.Credits) > 0 { opts = append(opts, query.WithCredits(filter.Credits)) } + if filter.SearchQuery != "" { + opts = append(opts, query.WithSearch(filter.SearchQuery)) + } return opts } diff --git a/service/review.go b/service/review.go index c18052f..fef6cbe 100644 --- a/service/review.go +++ b/service/review.go @@ -31,6 +31,9 @@ func buildReviewDBOptionFromFilter(query repository.IReviewQuery, filter domain. if filter.ReviewID != 0 { opts = append(opts, query.WithID(filter.ReviewID)) } + if filter.SearchQuery != "" { + opts = append(opts, query.WithSearch(filter.SearchQuery)) + } return opts } diff --git a/service/teacher.go b/service/teacher.go index b965c6d..8ec49b8 100644 --- a/service/teacher.go +++ b/service/teacher.go @@ -50,6 +50,9 @@ func buildTeacherDBOptionFromFilter(query repository.ITeacherQuery, filter domai if filter.PinyinAbbr != "" { opts = append(opts, query.WithPinyinAbbr(filter.PinyinAbbr)) } + if filter.SearchQuery != "" { + opts = append(opts, query.WithSearch(filter.SearchQuery)) + } opts = append(opts, query.WithPaginate(filter.Page, filter.PageSize)) return opts diff --git a/service/trainingplan.go b/service/trainingplan.go index 7410a4d..448619e 100644 --- a/service/trainingplan.go +++ b/service/trainingplan.go @@ -49,6 +49,9 @@ func buildTrainingPlanDBOptionFromFilter(query repository.ITrainingPlanQuery, fi if filter.Department != "" { opts = append(opts, query.WithDepartment(filter.Department)) } + if filter.SearchQuery != "" { + opts = append(opts, query.WithSearch(filter.SearchQuery)) + } opts = append(opts, query.WithPaginate(filter.Page, filter.PageSize)) return opts diff --git a/service/user.go b/service/user.go index 1e2272b..7b18eff 100644 --- a/service/user.go +++ b/service/user.go @@ -81,6 +81,9 @@ func buildUserDBOptionFromFilter(query repository.IUserQuery, filter domain.User if filter.Page > 0 { opts = append(opts, query.WithOffset(util.CalcOffset(filter.Page, filter.PageSize))) } + if filter.SearchQuery != "" { + opts = append(opts, query.WithSearch(filter.SearchQuery)) + } return opts } diff --git a/util/segword.go b/util/segword.go new file mode 100755 index 0000000..5266902 --- /dev/null +++ b/util/segword.go @@ -0,0 +1,22 @@ +package util + +import ( + "github.com/go-ego/gse" +) + +var seg gse.Segmenter + +func SegWord(src string) []string { + return seg.Slice(src, true) +} + +// TODO: add assets path +func InitSegWord() error { + if err := seg.LoadStop(); err != nil { + return err + } + if err := seg.LoadDict(); err != nil { + return err + } + return nil +} diff --git a/util/segword_test.go b/util/segword_test.go new file mode 100755 index 0000000..56f02a9 --- /dev/null +++ b/util/segword_test.go @@ -0,0 +1,16 @@ +package util_test + +import ( + "jcourse_go/util" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSegWord(t *testing.T) { + err := util.InitSegWord() + assert.NoError(t, err) + const txt = "电路理论" + var target = []string{"电路", "理论"} + assert.Equal(t, util.SegWord(txt), target) +} diff --git a/util/selenium-get/load2db_test.go b/util/selenium-get/load2db_test.go index 180708a..6e1004d 100644 --- a/util/selenium-get/load2db_test.go +++ b/util/selenium-get/load2db_test.go @@ -4,6 +4,7 @@ import ( "fmt" "jcourse_go/dal" "jcourse_go/model/po" + "jcourse_go/util" "reflect" "testing" ) @@ -25,6 +26,7 @@ func TestLoadTrainingPlan2DB(t *testing.T) { }) t.Run("mem db", func(t *testing.T) { dal.InitTestMemDBClient() + _ = util.InitSegWord() migrate() db := dal.GetDBClient() LoadTrainingPlan2DB("../../data/trainingPlan.txt", db) @@ -49,6 +51,7 @@ func TestLoadTeacherProfile2DB(t *testing.T) { t.Run("mem db", func(t *testing.T) { dal.InitTestMemDBClient() + _ = util.InitSegWord() migrate() db := dal.GetDBClient() LoadTeacherProfile2DB("../../data/teachers.json", db)