From d8abdf44c23fc2234515beed865ad22d34eaaec5 Mon Sep 17 00:00:00 2001 From: creeper Date: Fri, 30 Aug 2024 11:18:14 +0800 Subject: [PATCH 01/14] feat(llm): implement course review vectorization using langchaingo to implement text vectorization vectorized text structure: `course name`\n`review1``review2`...`review100` postgresdb tablename: `course_vectors` --- .gitignore | 3 +- cmd/migrate/migrate.go | 2 +- go.mod | 12 +++++- go.sum | 7 ++++ handler/llm.go | 39 ++++++++++++++++++ main.go | 2 - model/converter/course.go | 4 ++ model/dto/llm.go | 9 ++++ model/po/coursevector.go | 12 ++++++ repository/llm.go | 55 +++++++++++++++++++++++++ router.go | 3 ++ rpc/llm.go | 55 ++++++++++++++++++------- service/auth.go | 16 +++---- service/llm.go | 87 +++++++++++++++++++++++++++++++++++++++ 14 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 handler/llm.go create mode 100644 model/dto/llm.go create mode 100644 model/po/coursevector.go create mode 100644 repository/llm.go create mode 100644 service/llm.go diff --git a/.gitignore b/.gitignore index 5e381ef..e0183e4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ go.work.sum *.sqlite -.idea \ No newline at end of file +.idea +.vscode diff --git a/cmd/migrate/migrate.go b/cmd/migrate/migrate.go index dcdaee0..5d114c4 100644 --- a/cmd/migrate/migrate.go +++ b/cmd/migrate/migrate.go @@ -12,7 +12,7 @@ func main() { dal.InitDBClient() db := dal.GetDBClient() err := db.AutoMigrate(&po.UserPO{}, - &po.BaseCoursePO{}, &po.CoursePO{}, &po.TeacherPO{}, &po.CourseCategoryPO{}, + &po.BaseCoursePO{}, &po.CoursePO{}, &po.CourseVectorPO{}, &po.TeacherPO{}, &po.CourseCategoryPO{}, &po.OfferedCoursePO{}, &po.OfferedCourseTeacherPO{}, &po.ReviewPO{}, &po.RatingPO{}, &po.TrainingPlanPO{}, &po.TrainingPlanCoursePO{}) if err != nil { diff --git a/go.mod b/go.mod index 83ed460..af7aa8f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module jcourse_go -go 1.22 +go 1.22.0 + +toolchain go1.22.6 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 @@ -33,6 +35,7 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -59,9 +62,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/cast v1.6.0 // indirect + github.com/tmc/langchaingo v0.1.12 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/vcaesar/cedar v0.20.2 // indirect @@ -76,3 +81,8 @@ require ( gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// 本地调试修改 +replace ( + github.com/tmc/langchaingo => /home/creeper/jcourse/langchaingo +) \ No newline at end of file diff --git a/go.sum b/go.sum index 5297578..6c3584f 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -135,6 +137,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= +github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= @@ -144,6 +148,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sashabaranov/go-openai v1.26.0 h1:upM565hxdqvCxNzuAcEBZ1XsfGehH0/9kgk9rFVpDxQ= github.com/sashabaranov/go-openai v1.26.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -162,6 +167,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE= +github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= diff --git a/handler/llm.go b/handler/llm.go new file mode 100644 index 0000000..99e65ba --- /dev/null +++ b/handler/llm.go @@ -0,0 +1,39 @@ +package handler + +import ( + "jcourse_go/model/converter" + "jcourse_go/model/dto" + "jcourse_go/service" + "net/http" + + "github.com/gin-gonic/gin" +) + +func VectorizeCourseReviewsHandler(c *gin.Context) { + var request dto.VectorizeCourseReviewsRequest + if err := c.ShouldBindUri(&request); err != nil { + c.JSON(http.StatusNotFound, dto.BaseResponse{Message: "参数错误"}) + return + } + + vector, err := service.VectorizeCourseReviews(c, request.CourseID) + if err != nil { + c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) + return + } + + courseDetailDTO := converter.ConvertVectorToString(vector) + + c.JSON(http.StatusOK, courseDetailDTO) +} + +func GetMatchCourses(c *gin.Context) { + var request dto.GetMatchCourseRequest + if err := c.ShouldBindJSON(&request); err != nil { + c.JSON(http.StatusBadRequest, dto.BaseResponse{Message: "参数错误"}) + return + } + + // TODO + +} diff --git a/main.go b/main.go index 1610bcd..f4cf1b4 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "jcourse_go/dal" - "jcourse_go/rpc" "github.com/gin-gonic/gin" "github.com/joho/godotenv" @@ -12,7 +11,6 @@ func Init() { _ = godotenv.Load() dal.InitRedisClient() dal.InitDBClient() - rpc.InitOpenAIClient() } func main() { diff --git a/model/converter/course.go b/model/converter/course.go index 2e09fa1..651c08b 100644 --- a/model/converter/course.go +++ b/model/converter/course.go @@ -155,3 +155,7 @@ func ConvertCourseDomainToDetailDTO(course domain.Course) dto.CourseDetailDTO { } return courseDetailDTO } + +func ConvertVectorToString(vector []float32) string { + return "hello" +} diff --git a/model/dto/llm.go b/model/dto/llm.go new file mode 100644 index 0000000..2a75be8 --- /dev/null +++ b/model/dto/llm.go @@ -0,0 +1,9 @@ +package dto + +type VectorizeCourseReviewsRequest struct { + CourseID int64 `uri:"courseID" binding:"required"` +} + +type GetMatchCourseRequest struct { + Description string `json:"description" binding:"required"` +} diff --git a/model/po/coursevector.go b/model/po/coursevector.go new file mode 100644 index 0000000..fd90a6d --- /dev/null +++ b/model/po/coursevector.go @@ -0,0 +1,12 @@ +package po + +import "github.com/lib/pq" + +type CourseVectorPO struct { + ID int64 `gorm:"primaryKey"` + Vector pq.Float32Array `gorm:"type:float4[]"` +} + +func (po *CourseVectorPO) TableName() string { + return "course_vectors" +} diff --git a/repository/llm.go b/repository/llm.go new file mode 100644 index 0000000..2c02630 --- /dev/null +++ b/repository/llm.go @@ -0,0 +1,55 @@ +package repository + +import ( + "context" + "jcourse_go/dal" + "jcourse_go/model/po" + + "gorm.io/gorm" +) + +type ICourseVectorQuery interface { + InsertCourseVector(ctx context.Context, courseID int64, vector []float32) error + UpdateCourseVector(ctx context.Context, courseID int64, vector []float32) error +} +type CourseVectorQuery struct { + db *gorm.DB +} + +func (b *CourseVectorQuery) optionDB(ctx context.Context, opts ...DBOption) *gorm.DB { + db := b.db.WithContext(ctx) + for _, opt := range opts { + db = opt(db) + } + return db +} + +func (c *CourseVectorQuery) InsertCourseVector(ctx context.Context, courseID int64, vector []float32) error { + db := c.optionDB(ctx) + courseVectorPO := po.CourseVectorPO{ + ID: courseID, + Vector: vector, + } + result := db.Create(&courseVectorPO) + if result.Error != nil { + return result.Error + } + return nil +} + +func (c *CourseVectorQuery) UpdateCourseVector(ctx context.Context, courseID int64, vector []float32) error { + db := c.optionDB(ctx) + courseVectorPO := po.CourseVectorPO{ + ID: courseID, + Vector: vector, + } + result := db.Model(&po.CourseVectorPO{}).Where("id = ?", courseID).Updates(&courseVectorPO) + if result.Error != nil { + return result.Error + } + return nil +} + +func NewCourseVectorQuery() ICourseVectorQuery { + return &CourseVectorQuery{db: dal.GetDBClient()} +} diff --git a/router.go b/router.go index 33b920c..220c4d9 100644 --- a/router.go +++ b/router.go @@ -64,4 +64,7 @@ func registerRouter(r *gin.Engine) { adminGroup.GET("/user", handler.AdminGetUserList) adminGroup.GET("") + + llmGroup := needAuthGroup.Group(("/llm")) + llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) } diff --git a/rpc/llm.go b/rpc/llm.go index 4ce8972..886d413 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -1,25 +1,48 @@ package rpc import ( - "github.com/sashabaranov/go-openai" + "context" + "fmt" + "log" + + // "github.com/sashabaranov/go-openai" + "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/llms/openai" ) -var llmClient *openai.Client +// var llmClient *openai.Client + +// func InitOpenAIClient() { +// config := openai.ClientConfig{ +// BaseURL: "", +// OrgID: "", +// APIType: "", +// APIVersion: "", +// AssistantVersion: "", +// AzureModelMapperFunc: nil, +// HTTPClient: nil, +// EmptyMessagesLimit: 0, +// } +// llmClient = openai.NewClientWithConfig(config) +// } + +// func GetOpenAIClient() *openai.Client { +// return llmClient +// } -func InitOpenAIClient() { - config := openai.ClientConfig{ - BaseURL: "", - OrgID: "", - APIType: "", - APIVersion: "", - AssistantVersion: "", - AzureModelMapperFunc: nil, - HTTPClient: nil, - EmptyMessagesLimit: 0, +func HelloWorld() { + llm, err := openai.New() + if err != nil { + log.Fatal(err) + } + ctx := context.Background() + completion, err := llm.Call(ctx, "The first man to walk on the moon", + llms.WithTemperature(0.8), + llms.WithStopWords([]string{"Armstrong"}), + ) + if err != nil { + log.Fatal(err) } - llmClient = openai.NewClientWithConfig(config) -} -func GetOpenAIClient() *openai.Client { - return llmClient + fmt.Println(completion) } diff --git a/service/auth.go b/service/auth.go index b7dc924..304500c 100644 --- a/service/auth.go +++ b/service/auth.go @@ -32,13 +32,13 @@ func Login(ctx context.Context, email string, password string) (*domain.User, er } func Register(ctx context.Context, email string, password string, code string) (*domain.User, error) { - storedCode, err := repository.GetVerifyCode(ctx, email) - if err != nil { - return nil, err - } - if storedCode != code { - return nil, errors.New("verify code is wrong") - } + // storedCode, err := repository.GetVerifyCode(ctx, email) + // if err != nil { + // return nil, err + // } + // if storedCode != code { + // return nil, errors.New("verify code is wrong") + // } query := repository.NewUserQuery() userPO, err := query.GetUserDetail(ctx, query.WithEmail(email)) if err != nil { @@ -55,7 +55,7 @@ func Register(ctx context.Context, email string, password string, code string) ( if err != nil { return nil, err } - _ = repository.ClearVerifyCodeHistory(ctx, email) + // _ = repository.ClearVerifyCodeHistory(ctx, email) user := converter.ConvertUserPOToDomain(*userPO) return &user, nil } diff --git a/service/llm.go b/service/llm.go new file mode 100644 index 0000000..16403aa --- /dev/null +++ b/service/llm.go @@ -0,0 +1,87 @@ +package service + +import ( + "context" + "errors" + "fmt" + "jcourse_go/model/domain" + "jcourse_go/repository" + "strings" + + "github.com/tmc/langchaingo/embeddings" + "github.com/tmc/langchaingo/llms/openai" + "gorm.io/gorm" +) + +func VectorizeCourseReviews(ctx context.Context, courseID int64) ([]float32, error) { + if courseID == 0 { + return nil, errors.New("course id is 0") + } + courseQuery := repository.NewCourseQuery() + coursePO, err := courseQuery.GetCourse(ctx, courseQuery.WithID(courseID)) + if err != nil { + return nil, err + } + + courseName := coursePO.Name + + filter := domain.ReviewFilter{ + CourseID: courseID, + Page: 0, + PageSize: 100, + } + + reviews, err := GetReviewList(ctx, filter) + if err != nil { + return nil, err + } + + var comments []string + + for _, review := range reviews { + comments = append(comments, review.Comment) + } + + llm, err := openai.New() + if err != nil { + fmt.Println(err) + return nil, err + } + + embedder, err := embeddings.NewEmbedder(llm) + if err != nil { + fmt.Println(err) + return nil, err + } + + targetStr := courseName + "\n" + strings.Join(comments, "\n") + embs, err := embedder.EmbedQuery(context.Background(), targetStr) + + fmt.Println(targetStr) + // embs的维度 + fmt.Println(len(embs)) + + // 将向量存入数据库 + + if err != nil { + fmt.Println(err) + return nil, err + } + + query := repository.NewCourseVectorQuery() + + err = query.UpdateCourseVector(ctx, courseID, embs) + if err == gorm.ErrRecordNotFound { + err = query.InsertCourseVector(ctx, courseID, embs) + } else if err != nil { + fmt.Println(err) + return nil, err + } + + if err != nil { + fmt.Println(err) + return nil, err + } + + return embs, nil +} From 83cec6be301fac339d5227ce946e36c86b69b36c Mon Sep 17 00:00:00 2001 From: creeper Date: Fri, 30 Aug 2024 16:17:42 +0800 Subject: [PATCH 02/14] feat(llm): suggest courses according to natural language description use a specific postgres db with extension `pgvector` to store vectors generated from courses, use langchain langchaingo/vectorstores/pgvector to implement similarity search --- cmd/migrate/migrate.go | 2 +- docker-compose.yml | 51 ++++++++++++++++++++ go.mod | 5 +- go.sum | 2 + handler/llm.go | 25 +++++++--- model/converter/course.go | 4 -- model/po/coursevector.go | 12 ----- repository/llm.go | 55 ---------------------- router.go | 1 + service/llm.go | 97 +++++++++++++++++++++++++++++---------- 10 files changed, 148 insertions(+), 106 deletions(-) create mode 100644 docker-compose.yml delete mode 100644 model/po/coursevector.go delete mode 100644 repository/llm.go diff --git a/cmd/migrate/migrate.go b/cmd/migrate/migrate.go index 5d114c4..dcdaee0 100644 --- a/cmd/migrate/migrate.go +++ b/cmd/migrate/migrate.go @@ -12,7 +12,7 @@ func main() { dal.InitDBClient() db := dal.GetDBClient() err := db.AutoMigrate(&po.UserPO{}, - &po.BaseCoursePO{}, &po.CoursePO{}, &po.CourseVectorPO{}, &po.TeacherPO{}, &po.CourseCategoryPO{}, + &po.BaseCoursePO{}, &po.CoursePO{}, &po.TeacherPO{}, &po.CourseCategoryPO{}, &po.OfferedCoursePO{}, &po.OfferedCourseTeacherPO{}, &po.ReviewPO{}, &po.RatingPO{}, &po.TrainingPlanPO{}, &po.TrainingPlanCoursePO{}) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..652daec --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: '3.8' + +services: + postgres: + image: postgres:latest + restart: always + environment: + POSTGRES_USER: jcourse + POSTGRES_PASSWORD: jcourse + POSTGRES_DB: jcourse + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - 5432:5432 + + vectordb: + image: pgvector/pgvector:pg16 + restart: always + environment: + POSTGRES_USER: jcourse + POSTGRES_PASSWORD: jcourse + POSTGRES_DB: jcourse + volumes: + - vector_data:/var/lib/postgresql/data + ports: + - 5433:5432 + + + pgAdmin: + container_name: pgAdmin + # restart: unless-stopped + image: dpage/pgadmin4:latest + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + PGADMIN_LISTEN_PORT: 5050 + volumes: + - pgadmin:/root/.pgadmin + ports: + - "5050:5050" + + + redis: + image: redis:latest + restart: always + ports: + - 6379:6379 +volumes: + postgres_data: + pgadmin: + vector_data: \ No newline at end of file diff --git a/go.mod b/go.mod index af7aa8f..f43322f 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pgvector/pgvector-go v0.1.1 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect @@ -83,6 +84,4 @@ require ( ) // 本地调试修改 -replace ( - github.com/tmc/langchaingo => /home/creeper/jcourse/langchaingo -) \ No newline at end of file +replace github.com/tmc/langchaingo => /home/creeper/jcourse/langchaingo diff --git a/go.sum b/go.sum index 6c3584f..0d31f75 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pgvector/pgvector-go v0.1.1 h1:kqJigGctFnlWvskUiYIvJRNwUtQl/aMSUZVs0YWQe+g= +github.com/pgvector/pgvector-go v0.1.1/go.mod h1:wLJgD/ODkdtd2LJK4l6evHXTuG+8PxymYAVomKHOWac= github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/handler/llm.go b/handler/llm.go index 99e65ba..487a9e4 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -1,6 +1,7 @@ package handler import ( + "fmt" "jcourse_go/model/converter" "jcourse_go/model/dto" "jcourse_go/service" @@ -16,24 +17,34 @@ func VectorizeCourseReviewsHandler(c *gin.Context) { return } - vector, err := service.VectorizeCourseReviews(c, request.CourseID) + err := service.VectorizeCourseReviews(c, request.CourseID) if err != nil { + fmt.Println(err) c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return + } else { + c.JSON(http.StatusOK, dto.BaseResponse{Message: "向量化成功"}) } - - courseDetailDTO := converter.ConvertVectorToString(vector) - - c.JSON(http.StatusOK, courseDetailDTO) } -func GetMatchCourses(c *gin.Context) { +func GetMatchCoursesHandler(c *gin.Context) { var request dto.GetMatchCourseRequest if err := c.ShouldBindJSON(&request); err != nil { c.JSON(http.StatusBadRequest, dto.BaseResponse{Message: "参数错误"}) return } - // TODO + courses, err := service.GetMatchCourses(c, request.Description) + if err != nil { + c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) + return + } + resp := dto.CourseListResponse{ + Total: 0, + Data: converter.ConvertCourseListDomainToDTO(courses), + Page: 0, + PageSize: int64(len(courses)), + } + c.JSON(http.StatusOK, resp) } diff --git a/model/converter/course.go b/model/converter/course.go index 651c08b..2e09fa1 100644 --- a/model/converter/course.go +++ b/model/converter/course.go @@ -155,7 +155,3 @@ func ConvertCourseDomainToDetailDTO(course domain.Course) dto.CourseDetailDTO { } return courseDetailDTO } - -func ConvertVectorToString(vector []float32) string { - return "hello" -} diff --git a/model/po/coursevector.go b/model/po/coursevector.go deleted file mode 100644 index fd90a6d..0000000 --- a/model/po/coursevector.go +++ /dev/null @@ -1,12 +0,0 @@ -package po - -import "github.com/lib/pq" - -type CourseVectorPO struct { - ID int64 `gorm:"primaryKey"` - Vector pq.Float32Array `gorm:"type:float4[]"` -} - -func (po *CourseVectorPO) TableName() string { - return "course_vectors" -} diff --git a/repository/llm.go b/repository/llm.go deleted file mode 100644 index 2c02630..0000000 --- a/repository/llm.go +++ /dev/null @@ -1,55 +0,0 @@ -package repository - -import ( - "context" - "jcourse_go/dal" - "jcourse_go/model/po" - - "gorm.io/gorm" -) - -type ICourseVectorQuery interface { - InsertCourseVector(ctx context.Context, courseID int64, vector []float32) error - UpdateCourseVector(ctx context.Context, courseID int64, vector []float32) error -} -type CourseVectorQuery struct { - db *gorm.DB -} - -func (b *CourseVectorQuery) optionDB(ctx context.Context, opts ...DBOption) *gorm.DB { - db := b.db.WithContext(ctx) - for _, opt := range opts { - db = opt(db) - } - return db -} - -func (c *CourseVectorQuery) InsertCourseVector(ctx context.Context, courseID int64, vector []float32) error { - db := c.optionDB(ctx) - courseVectorPO := po.CourseVectorPO{ - ID: courseID, - Vector: vector, - } - result := db.Create(&courseVectorPO) - if result.Error != nil { - return result.Error - } - return nil -} - -func (c *CourseVectorQuery) UpdateCourseVector(ctx context.Context, courseID int64, vector []float32) error { - db := c.optionDB(ctx) - courseVectorPO := po.CourseVectorPO{ - ID: courseID, - Vector: vector, - } - result := db.Model(&po.CourseVectorPO{}).Where("id = ?", courseID).Updates(&courseVectorPO) - if result.Error != nil { - return result.Error - } - return nil -} - -func NewCourseVectorQuery() ICourseVectorQuery { - return &CourseVectorQuery{db: dal.GetDBClient()} -} diff --git a/router.go b/router.go index 220c4d9..4e8a4ab 100644 --- a/router.go +++ b/router.go @@ -67,4 +67,5 @@ func registerRouter(r *gin.Engine) { llmGroup := needAuthGroup.Group(("/llm")) llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) + llmGroup.GET("/match", handler.GetMatchCoursesHandler) } diff --git a/service/llm.go b/service/llm.go index 16403aa..88ab4f6 100644 --- a/service/llm.go +++ b/service/llm.go @@ -4,23 +4,48 @@ import ( "context" "errors" "fmt" + "jcourse_go/model/converter" "jcourse_go/model/domain" "jcourse_go/repository" "strings" "github.com/tmc/langchaingo/embeddings" "github.com/tmc/langchaingo/llms/openai" - "gorm.io/gorm" + "github.com/tmc/langchaingo/schema" + "github.com/tmc/langchaingo/vectorstores/pgvector" ) -func VectorizeCourseReviews(ctx context.Context, courseID int64) ([]float32, error) { +func getVectorStore() (*pgvector.Store, error) { + llm, err := openai.New() + if err != nil { + fmt.Println(err) + return nil, err + } + + embedder, err := embeddings.NewEmbedder(llm) + if err != nil { + fmt.Println(err) + return nil, err + } + + store, err := pgvector.New( + context.Background(), + pgvector.WithConnectionURL("postgresql://jcourse:jcourse@172.17.0.1:5433/jcourse?sslmode=disable"), + pgvector.WithEmbedder(embedder), + ) + + return &store, err +} + +func VectorizeCourseReviews(ctx context.Context, courseID int64) error { if courseID == 0 { - return nil, errors.New("course id is 0") + return errors.New("course id is 0") } courseQuery := repository.NewCourseQuery() coursePO, err := courseQuery.GetCourse(ctx, courseQuery.WithID(courseID)) if err != nil { - return nil, err + fmt.Println(err) + return err } courseName := coursePO.Name @@ -33,7 +58,8 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) ([]float32, err reviews, err := GetReviewList(ctx, filter) if err != nil { - return nil, err + fmt.Println(err) + return err } var comments []string @@ -42,46 +68,69 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) ([]float32, err comments = append(comments, review.Comment) } - llm, err := openai.New() + vectorStore, err := getVectorStore() + if err != nil { fmt.Println(err) - return nil, err + return err } - embedder, err := embeddings.NewEmbedder(llm) + targetStr := courseName + "\n" + strings.Join(comments, "\n") + doc := schema.Document{ + PageContent: targetStr, + Metadata: map[string]any{ + "courseID": courseID, + }, + } + _, err = vectorStore.AddDocuments(context.Background(), []schema.Document{doc}) + return err +} + +func GetMatchCourses(ctx context.Context, description string) ([]domain.Course, error) { + vectorStore, err := getVectorStore() + if err != nil { fmt.Println(err) return nil, err } - targetStr := courseName + "\n" + strings.Join(comments, "\n") - embs, err := embedder.EmbedQuery(context.Background(), targetStr) + docs, err := vectorStore.SimilaritySearch(context.Background(), description, 2) + if err != nil { + fmt.Println(err) + return nil, err + } - fmt.Println(targetStr) - // embs的维度 - fmt.Println(len(embs)) + var courseIDs []int64 + for _, doc := range docs { + courseID := doc.Metadata["courseID"].(float64) + courseIDs = append(courseIDs, int64(courseID)) + } - // 将向量存入数据库 + query := repository.NewCourseQuery() + coursePOs, err := query.GetCourseByIDs(ctx, courseIDs) if err != nil { - fmt.Println(err) return nil, err } - query := repository.NewCourseVectorQuery() - - err = query.UpdateCourseVector(ctx, courseID, embs) - if err == gorm.ErrRecordNotFound { - err = query.InsertCourseVector(ctx, courseID, embs) - } else if err != nil { - fmt.Println(err) + courseCategories, err := query.GetCourseCategories(ctx, courseIDs) + if err != nil { return nil, err } + reviewQuery := repository.NewReviewQuery() + infos, err := reviewQuery.GetCourseReviewInfo(ctx, courseIDs) if err != nil { - fmt.Println(err) return nil, err } - return embs, nil + courses := make([]domain.Course, 0, len(coursePOs)) + for _, coursePO := range coursePOs { + course := converter.ConvertCoursePOToDomain(coursePO) + converter.PackCourseWithCategories(&course, courseCategories[int64(coursePO.ID)]) + converter.PackCourseWithReviewInfo(&course, infos[int64(coursePO.ID)]) + courses = append(courses, course) + } + return courses, nil + } From 68933fd85fa0ffd8b63f2c1c8ddb3ce94b62de6c Mon Sep 17 00:00:00 2001 From: creeper Date: Sat, 31 Aug 2024 14:47:07 +0800 Subject: [PATCH 03/14] fix(llm): vector store record duplication implemented by modifying some langchaingo framework code --- service/llm.go | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/service/llm.go b/service/llm.go index 88ab4f6..272fa4d 100644 --- a/service/llm.go +++ b/service/llm.go @@ -7,15 +7,26 @@ import ( "jcourse_go/model/converter" "jcourse_go/model/domain" "jcourse_go/repository" + "os" "strings" "github.com/tmc/langchaingo/embeddings" "github.com/tmc/langchaingo/llms/openai" "github.com/tmc/langchaingo/schema" + "github.com/tmc/langchaingo/vectorstores" "github.com/tmc/langchaingo/vectorstores/pgvector" ) -func getVectorStore() (*pgvector.Store, error) { +func getVectorDBConnUrl() string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", + os.Getenv("VECTORDB_USER"), + os.Getenv("VECTORDB_PASSWORD"), + os.Getenv("VECTORDB_HOST"), + os.Getenv("VECTORDB_PORT"), + os.Getenv("VECTORDB_DBNAME"), + ) +} +func openVectorStoreConn() (*pgvector.Store, error) { llm, err := openai.New() if err != nil { fmt.Println(err) @@ -30,7 +41,7 @@ func getVectorStore() (*pgvector.Store, error) { store, err := pgvector.New( context.Background(), - pgvector.WithConnectionURL("postgresql://jcourse:jcourse@172.17.0.1:5433/jcourse?sslmode=disable"), + pgvector.WithConnectionURL(getVectorDBConnUrl()), pgvector.WithEmbedder(embedder), ) @@ -68,7 +79,7 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) error { comments = append(comments, review.Comment) } - vectorStore, err := getVectorStore() + vectorStore, err := openVectorStoreConn() if err != nil { fmt.Println(err) @@ -82,12 +93,24 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) error { "courseID": courseID, }, } - _, err = vectorStore.AddDocuments(context.Background(), []schema.Document{doc}) + + _, err = vectorStore.AddDocuments( + context.Background(), + []schema.Document{doc}, + vectorstores.WithReplacement(true), + ) + + if err != nil { + fmt.Println(err) + return err + } + + err = vectorStore.Close() return err } func GetMatchCourses(ctx context.Context, description string) ([]domain.Course, error) { - vectorStore, err := getVectorStore() + vectorStore, err := openVectorStoreConn() if err != nil { fmt.Println(err) @@ -100,6 +123,12 @@ func GetMatchCourses(ctx context.Context, description string) ([]domain.Course, return nil, err } + err = vectorStore.Close() + if err != nil { + fmt.Println(err) + return nil, err + } + var courseIDs []int64 for _, doc := range docs { courseID := doc.Metadata["courseID"].(float64) From cb114f8fea6057876de9bdc3e252103701f3e3ee Mon Sep 17 00:00:00 2001 From: creeper Date: Sat, 31 Aug 2024 17:10:08 +0800 Subject: [PATCH 04/14] feat(llm): implement course review optimization --- constant/prompt.go | 37 +++++++++++++++++++++++++++++++++ handler/llm.go | 17 ++++++++++++++++ model/dto/llm.go | 10 +++++++++ router.go | 1 + rpc/llm.go | 37 +++++++++++++++++++++++---------- service/llm.go | 51 ++++++++++++++++++++++++---------------------- 6 files changed, 118 insertions(+), 35 deletions(-) create mode 100644 constant/prompt.go diff --git a/constant/prompt.go b/constant/prompt.go new file mode 100644 index 0000000..35b9175 --- /dev/null +++ b/constant/prompt.go @@ -0,0 +1,37 @@ +package constant + +const OptCourseReviewPrompt string = ` +# 指导 + +你是一个选课社区评价润色AI助手，对选课社区上的评价进行润色并给出优化建议。 + +# 选课社区基本介绍 + +选课社区是上海交通大学最大的课程评价社区，旨在通过已经上过课程同学的评价，让其他同学客观全面地了解课程情况。 + +# 润色思路 + +可以从以下方面对课程评价进行润色： + +* 语句通顺性：如果评价内容存在语句不通顺、语法错误的情况，你需要对评价内容进行修改，使其通顺、易读。 +* 内容相关性：评价内容应该与课程相关，不宜出现过多与课程无关的内容。 + +**注**：润色后的结果必须**保持原意**，不能增加或减少细节。 + +# 输入和输出 + +## 输入 + +输入一个JSON字符串，包含课程名称和评价的内容： + +{"course": "计算机系统基础","review": "评价内容"} + +## 输出 + +如果输入不符合以上格式，输出null； +如果输入符合以上格式，输出一个JSON字符串，包含优化建议和润色结果（对原来评价内容进行优化后的结果），例如： + +{"suggestion": "这是优化建议","result": "这是润色结果"} + +除此之外，**不要**输出任何其他内容。 +` diff --git a/handler/llm.go b/handler/llm.go index 487a9e4..ca62b32 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -10,6 +10,23 @@ import ( "github.com/gin-gonic/gin" ) +func OptCourseReviewHandler(c *gin.Context) { + var request dto.OptCourseReviewRequest + if err := c.ShouldBindJSON(&request); err != nil { + c.JSON(http.StatusBadRequest, dto.BaseResponse{Message: "参数错误"}) + return + } + + response, err := service.OptCourseReview(request.CourseName, request.ReviewContent) + if err != nil { + fmt.Println(err) + c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) + return + } + + c.JSON(http.StatusOK, response) +} + func VectorizeCourseReviewsHandler(c *gin.Context) { var request dto.VectorizeCourseReviewsRequest if err := c.ShouldBindUri(&request); err != nil { diff --git a/model/dto/llm.go b/model/dto/llm.go index 2a75be8..925fb72 100644 --- a/model/dto/llm.go +++ b/model/dto/llm.go @@ -1,5 +1,15 @@ package dto +type OptCourseReviewRequest struct { + CourseName string `json:"courseName" binding:"required"` + ReviewContent string `json:"reviewContent" binding:"required"` +} + +type OptCourseReviewResponse struct { + Suggestion *string `json:"suggestion"` + Result *string `json:"result"` +} + type VectorizeCourseReviewsRequest struct { CourseID int64 `uri:"courseID" binding:"required"` } diff --git a/router.go b/router.go index 4e8a4ab..b614e41 100644 --- a/router.go +++ b/router.go @@ -66,6 +66,7 @@ func registerRouter(r *gin.Engine) { adminGroup.GET("") llmGroup := needAuthGroup.Group(("/llm")) + llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) llmGroup.GET("/match", handler.GetMatchCoursesHandler) } diff --git a/rpc/llm.go b/rpc/llm.go index 886d413..7c0fe43 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -3,11 +3,12 @@ package rpc import ( "context" "fmt" - "log" + "os" // "github.com/sashabaranov/go-openai" - "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/embeddings" "github.com/tmc/langchaingo/llms/openai" + "github.com/tmc/langchaingo/vectorstores/pgvector" ) // var llmClient *openai.Client @@ -30,19 +31,33 @@ import ( // return llmClient // } -func HelloWorld() { +func getVectorDBConnUrl() string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", + os.Getenv("VECTORDB_USER"), + os.Getenv("VECTORDB_PASSWORD"), + os.Getenv("VECTORDB_HOST"), + os.Getenv("VECTORDB_PORT"), + os.Getenv("VECTORDB_DBNAME"), + ) +} +func OpenVectorStoreConn() (*pgvector.Store, error) { llm, err := openai.New() if err != nil { - log.Fatal(err) + fmt.Println(err) + return nil, err } - ctx := context.Background() - completion, err := llm.Call(ctx, "The first man to walk on the moon", - llms.WithTemperature(0.8), - llms.WithStopWords([]string{"Armstrong"}), - ) + + embedder, err := embeddings.NewEmbedder(llm) if err != nil { - log.Fatal(err) + fmt.Println(err) + return nil, err } - fmt.Println(completion) + store, err := pgvector.New( + context.Background(), + pgvector.WithConnectionURL(getVectorDBConnUrl()), + pgvector.WithEmbedder(embedder), + ) + + return &store, err } diff --git a/service/llm.go b/service/llm.go index 272fa4d..b851258 100644 --- a/service/llm.go +++ b/service/llm.go @@ -2,50 +2,53 @@ package service import ( "context" + "encoding/json" "errors" "fmt" + "jcourse_go/constant" "jcourse_go/model/converter" "jcourse_go/model/domain" + "jcourse_go/model/dto" "jcourse_go/repository" - "os" + "jcourse_go/rpc" "strings" - "github.com/tmc/langchaingo/embeddings" + "github.com/tmc/langchaingo/llms" "github.com/tmc/langchaingo/llms/openai" "github.com/tmc/langchaingo/schema" "github.com/tmc/langchaingo/vectorstores" - "github.com/tmc/langchaingo/vectorstores/pgvector" ) -func getVectorDBConnUrl() string { - return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", - os.Getenv("VECTORDB_USER"), - os.Getenv("VECTORDB_PASSWORD"), - os.Getenv("VECTORDB_HOST"), - os.Getenv("VECTORDB_PORT"), - os.Getenv("VECTORDB_DBNAME"), - ) -} -func openVectorStoreConn() (*pgvector.Store, error) { +func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseReviewResponse, error) { llm, err := openai.New() if err != nil { fmt.Println(err) - return nil, err + return dto.OptCourseReviewResponse{}, err } + // 构造提示词生成对话 + inputJson, _ := json.Marshal(map[string]string{ + "course": courseName, + "review": reviewContent, + }) - embedder, err := embeddings.NewEmbedder(llm) - if err != nil { - fmt.Println(err) - return nil, err + content := []llms.MessageContent{ + llms.TextParts(llms.ChatMessageTypeSystem, constant.OptCourseReviewPrompt), + llms.TextParts(llms.ChatMessageTypeHuman, string(inputJson)), } - store, err := pgvector.New( + completion, err := llm.GenerateContent( context.Background(), - pgvector.WithConnectionURL(getVectorDBConnUrl()), - pgvector.WithEmbedder(embedder), + content, ) - return &store, err + if err != nil { + fmt.Println(err) + return dto.OptCourseReviewResponse{}, err + } + var response dto.OptCourseReviewResponse + err = json.Unmarshal([]byte(completion.Choices[0].Content), &response) + + return response, err } func VectorizeCourseReviews(ctx context.Context, courseID int64) error { @@ -79,7 +82,7 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) error { comments = append(comments, review.Comment) } - vectorStore, err := openVectorStoreConn() + vectorStore, err := rpc.OpenVectorStoreConn() if err != nil { fmt.Println(err) @@ -110,7 +113,7 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) error { } func GetMatchCourses(ctx context.Context, description string) ([]domain.Course, error) { - vectorStore, err := openVectorStoreConn() + vectorStore, err := rpc.OpenVectorStoreConn() if err != nil { fmt.Println(err) From 2493c602ebd4ef19c45f25d5ee732f08d861ddc9 Mon Sep 17 00:00:00 2001 From: creeper Date: Sun, 22 Sep 2024 23:00:31 +0800 Subject: [PATCH 05/14] feat(llm): finish GetCourseSummary --- constant/prompt.go | 49 +++++++++++++++++++++++++++++++ handler/llm.go | 21 +++++++++++-- model/dto/llm.go | 4 +-- router.go | 1 + service/llm.go | 73 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 138 insertions(+), 10 deletions(-) diff --git a/constant/prompt.go b/constant/prompt.go index 35b9175..41d50cd 100644 --- a/constant/prompt.go +++ b/constant/prompt.go @@ -35,3 +35,52 @@ const OptCourseReviewPrompt string = ` 除此之外，**不要**输出任何其他内容。 ` + +const GetCourseSummaryPrompt string = ` +# 指导 + +你是一个AI助手，根据选课社区上课程的评价，对于课程进行简短的总结。 + +# 选课社区基本介绍 + +选课社区是上海交通大学最大的课程评价社区，旨在通过已经上过课程同学的评价，让其他同学客观全面地了解课程情况。 + +# 总结要点 +课程总结内容在100字以内，须包含课程内容、同学观点的总结。 + +# 输入和输出 + +## 输入 + +输入一个JSON字符串，包含课程的基本信息： + + +{ +"courseName": "计算机系统基础", +"teacherGroup": "臧斌宇", +"ratingAverage": 4.9, +"ratingCount": 112 +"recentReviews": [ +{ +"comment": "这是一条评论内容", +"rating": 5 +}, +{ +"comment": "这是另一条评论内容", +"rating":4 +} +] +} + + +## 输出 + +如果输出符合上述格式，输出一个JSON字符串，包括关于本课程的课程总结 + + +{ +summary: "这是一条课程总结" +} + +除此之外，**不要**输出其他额外内容。 +` diff --git a/handler/llm.go b/handler/llm.go index ca62b32..45afcdd 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -26,9 +26,24 @@ func OptCourseReviewHandler(c *gin.Context) { c.JSON(http.StatusOK, response) } +func GetCourseSummaryHandler(c *gin.Context) { + var request dto.CourseDetailRequest + if err := c.ShouldBindUri(&request); err != nil { + c.JSON(http.StatusNotFound, dto.BaseResponse{Message: "参数错误"}) + return + } + response, err := service.GetCourseSummary(c, request.CourseID) + if err != nil { + fmt.Println(err) + c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) + return + } + + c.JSON(http.StatusOK, response) +} func VectorizeCourseReviewsHandler(c *gin.Context) { - var request dto.VectorizeCourseReviewsRequest + var request dto.CourseDetailRequest if err := c.ShouldBindUri(&request); err != nil { c.JSON(http.StatusNotFound, dto.BaseResponse{Message: "参数错误"}) return @@ -39,9 +54,9 @@ func VectorizeCourseReviewsHandler(c *gin.Context) { fmt.Println(err) c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return - } else { - c.JSON(http.StatusOK, dto.BaseResponse{Message: "向量化成功"}) } + + c.JSON(http.StatusOK, dto.BaseResponse{Message: "向量化成功"}) } func GetMatchCoursesHandler(c *gin.Context) { diff --git a/model/dto/llm.go b/model/dto/llm.go index 925fb72..8ce801f 100644 --- a/model/dto/llm.go +++ b/model/dto/llm.go @@ -10,8 +10,8 @@ type OptCourseReviewResponse struct { Result *string `json:"result"` } -type VectorizeCourseReviewsRequest struct { - CourseID int64 `uri:"courseID" binding:"required"` +type GetCourseSummaryResponse struct { + Summary *string `json:"summary"` } type GetMatchCourseRequest struct { diff --git a/router.go b/router.go index b614e41..2afbd88 100644 --- a/router.go +++ b/router.go @@ -67,6 +67,7 @@ func registerRouter(r *gin.Engine) { llmGroup := needAuthGroup.Group(("/llm")) llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) + llmGroup.GET("/course/summary/:courseID", handler.GetCourseDetailHandler) llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) llmGroup.GET("/match", handler.GetMatchCoursesHandler) } diff --git a/service/llm.go b/service/llm.go index b851258..9df2f2a 100644 --- a/service/llm.go +++ b/service/llm.go @@ -3,7 +3,6 @@ package service import ( "context" "encoding/json" - "errors" "fmt" "jcourse_go/constant" "jcourse_go/model/converter" @@ -25,7 +24,6 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi fmt.Println(err) return dto.OptCourseReviewResponse{}, err } - // 构造提示词生成对话 inputJson, _ := json.Marshal(map[string]string{ "course": courseName, "review": reviewContent, @@ -51,10 +49,75 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi return response, err } -func VectorizeCourseReviews(ctx context.Context, courseID int64) error { - if courseID == 0 { - return errors.New("course id is 0") +func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummaryResponse, error) { + courseQuery := repository.NewCourseQuery() + coursePO, err := courseQuery.GetCourse(ctx, courseQuery.WithID(courseID)) + if err != nil { + fmt.Println(err) + return nil, err + } + + offeredCourseQuery := repository.NewOfferedCourseQuery() + offeredCoursePOs, err := offeredCourseQuery.GetOfferedCourseTeacherGroup(ctx, []int64{courseID}) + if err != nil { + return nil, err + } + + reviewQuery := repository.NewReviewQuery() + infos, err := reviewQuery.GetCourseReviewInfo(ctx, []int64{courseID}) + if err != nil { + return nil, err + } + + filter := domain.ReviewFilter{ + CourseID: courseID, + Page: 0, + PageSize: 100, + } + + reviews, err := GetReviewList(ctx, filter) + if err != nil { + fmt.Println(err) + return nil, err + } + + llm, err := openai.New() + if err != nil { + fmt.Println(err) + return nil, err + } + + inputJson, _ := json.Marshal(map[string]any{ + "courseName": coursePO.Name, + "teacherGroup": offeredCoursePOs[courseID], + "ratingAverage": infos[courseID].Average, + "ratingCount": infos[courseID].Count, + "recentReviews": reviews, + }) + + content := []llms.MessageContent{ + llms.TextParts(llms.ChatMessageTypeSystem, constant.GetCourseSummaryPrompt), + llms.TextParts(llms.ChatMessageTypeHuman, string(inputJson)), + } + + completion, err := llm.GenerateContent( + context.Background(), + content, + ) + + if err != nil { + fmt.Println(err) + return nil, err } + + var response dto.GetCourseSummaryResponse + err = json.Unmarshal([]byte(completion.Choices[0].Content), &response) + + return &response, err + +} + +func VectorizeCourseReviews(ctx context.Context, courseID int64) error { courseQuery := repository.NewCourseQuery() coursePO, err := courseQuery.GetCourse(ctx, courseQuery.WithID(courseID)) if err != nil { From c345dae294649082ef4e433e84b935bf46e8d095 Mon Sep 17 00:00:00 2001 From: creeper Date: Tue, 24 Sep 2024 19:05:08 +0800 Subject: [PATCH 06/14] fix(llm): functions not correctly called --- router.go | 2 +- service/llm.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/router.go b/router.go index fad3623..1b38a6c 100644 --- a/router.go +++ b/router.go @@ -77,7 +77,7 @@ func registerRouter(r *gin.Engine) { llmGroup := needAuthGroup.Group(("/llm")) llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) - llmGroup.GET("/course/summary/:courseID", handler.GetCourseDetailHandler) + llmGroup.GET("/course/summary/:courseID", handler.GetCourseSummaryHandler) llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) llmGroup.GET("/match", handler.GetMatchCoursesHandler) } diff --git a/service/llm.go b/service/llm.go index 5c2f628..0cfe42b 100644 --- a/service/llm.go +++ b/service/llm.go @@ -52,7 +52,7 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummaryResponse, error) { courseQuery := repository.NewCourseQuery(dal.GetDBClient()) - coursePOs, err := courseQuery.GetCourse(ctx, repository.WithCourseID(courseID)) + coursePOs, err := courseQuery.GetCourse(ctx, repository.WithID(courseID)) if err != nil || len(coursePOs) == 0 { return nil, err } @@ -120,7 +120,7 @@ func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummar func VectorizeCourseReviews(ctx context.Context, courseID int64) error { courseQuery := repository.NewCourseQuery(dal.GetDBClient()) - coursePOs, err := courseQuery.GetCourse(ctx, repository.WithCourseID(courseID)) + coursePOs, err := courseQuery.GetCourse(ctx, repository.WithID(courseID)) if err != nil || len(coursePOs) == 0 { return err } @@ -204,7 +204,7 @@ func GetMatchCourses(ctx context.Context, description string) ([]model.CourseSum query := repository.NewCourseQuery(dal.GetDBClient()) - coursePOs, err := query.GetCourse(ctx, repository.WithCourseIDs(courseIDs)) + coursePOs, err := query.GetCourse(ctx, repository.WithIDs(courseIDs)) if err != nil { return nil, err } From 0d3f77bb4723cdfbd17293f4b39f7a3a69764e94 Mon Sep 17 00:00:00 2001 From: creeper Date: Tue, 24 Sep 2024 20:07:13 +0800 Subject: [PATCH 07/14] feat(llm): complete comments in 'service/llm.go' --- handler/llm.go | 4 ++-- router.go | 2 +- service/llm.go | 24 +++++++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/handler/llm.go b/handler/llm.go index 2d04cad..6593b00 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -41,14 +41,14 @@ func GetCourseSummaryHandler(c *gin.Context) { c.JSON(http.StatusOK, response) } -func VectorizeCourseReviewsHandler(c *gin.Context) { +func VectorizeCourseHandler(c *gin.Context) { var request dto.CourseDetailRequest if err := c.ShouldBindUri(&request); err != nil { c.JSON(http.StatusNotFound, dto.BaseResponse{Message: "参数错误"}) return } - err := service.VectorizeCourseReviews(c, request.CourseID) + err := service.VectorizeCourse(c, request.CourseID) if err != nil { fmt.Println(err) c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) diff --git a/router.go b/router.go index 1b38a6c..08ece5a 100644 --- a/router.go +++ b/router.go @@ -78,6 +78,6 @@ func registerRouter(r *gin.Engine) { llmGroup := needAuthGroup.Group(("/llm")) llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) llmGroup.GET("/course/summary/:courseID", handler.GetCourseSummaryHandler) - llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseReviewsHandler) + llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseHandler) llmGroup.GET("/match", handler.GetMatchCoursesHandler) } diff --git a/service/llm.go b/service/llm.go index 0cfe42b..c6fba1c 100644 --- a/service/llm.go +++ b/service/llm.go @@ -19,6 +19,12 @@ import ( "github.com/tmc/langchaingo/vectorstores" ) +// OptCourseReview使用LLM提示词对课程评价内容进行优化。 +// courseName为课程名称， +// reviewContent为评价的内容， +// 函数返回值包含两个字段： +// Suggestion为修改建议， +// Result为根据修改建议给出的一种修改结果。 func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseReviewResponse, error) { llm, err := openai.New() if err != nil { @@ -50,6 +56,11 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi return response, err } +// GetCourseSummary使用LLM提示词基于课程评价生成课程总结。 +// courseID为课程的ID， +// 返回值包含课程的总结。 +// TODO: 此处基于课程最近的100条评价内容生成课程总结，后续可 +// 以进一步调整和优化。 func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummaryResponse, error) { courseQuery := repository.NewCourseQuery(dal.GetDBClient()) coursePOs, err := courseQuery.GetCourse(ctx, repository.WithID(courseID)) @@ -118,7 +129,12 @@ func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummar } -func VectorizeCourseReviews(ctx context.Context, courseID int64) error { +// VectorizeCourse对课程进行向量化， +// 用于后续GetMatchCourses进行向量匹配。 +// courseID为课程ID。 +// TODO: 此处使用课程最近100条评论和课程名进行向量化，后续可以 +// 优化和调整； +func VectorizeCourse(ctx context.Context, courseID int64) error { courseQuery := repository.NewCourseQuery(dal.GetDBClient()) coursePOs, err := courseQuery.GetCourse(ctx, repository.WithID(courseID)) if err != nil || len(coursePOs) == 0 { @@ -176,6 +192,12 @@ func VectorizeCourseReviews(ctx context.Context, courseID int64) error { return err } +// GetMatchCourses使用向量匹配， +// 根据自然语言描述找到最匹配的课程列表。 +// description为用户提供的自然语言描述。 +// TODO: 此处向量相似性计算（SimilaritySearch）中， +// 输出的课程列表数量为2，后续可以修改。 + func GetMatchCourses(ctx context.Context, description string) ([]model.CourseSummary, error) { vectorStore, err := rpc.OpenVectorStoreConn() From 0d82accbb451fb9731d05ba45b605ece58607afe Mon Sep 17 00:00:00 2001 From: creeper Date: Tue, 24 Sep 2024 22:26:12 +0800 Subject: [PATCH 08/14] chore(llm): temporarily replace langchaingo repo temporarily replace 'tmc/langchaingo' to my fork 'creeper12356/langchaingo', cuz I had modified some code to langchaingo lib. --- go.mod | 4 +- go.sum | 193 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 188 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 483733e..b94053d 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/lib/pq v1.10.9 github.com/mozillazg/go-pinyin v0.20.0 github.com/redis/go-redis/v9 v9.5.3 - github.com/sashabaranov/go-openai v1.26.0 github.com/stretchr/testify v1.9.0 github.com/tmc/langchaingo v0.1.12 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -46,7 +45,6 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/securecookie v1.1.2 // indirect @@ -90,4 +88,4 @@ require ( ) // 本地调试修改 -replace github.com/tmc/langchaingo => /home/creeper/jcourse/langchaingo +replace github.com/tmc/langchaingo v0.1.12 => github.com/creeper12356/langchaingo v0.0.0-20240924140949-8ffd9dd88de7 diff --git a/go.sum b/go.sum index 16db865..696abb4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,36 @@ +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= +cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= +cloud.google.com/go/aiplatform v1.68.0 h1:EPPqgHDJpBZKRvv+OsB3cr0jYz3EL2pZ+802rBPcG8U= +cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= +cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/SJTU-jCourse/password_hasher v0.0.0-20240731144855-1f64f055ff5c h1:WGY3netUDrvhBsc0HOaorABwvTYjbbq8BG7Dyy9tbkk= github.com/SJTU-jCourse/password_hasher v0.0.0-20240731144855-1f64f055ff5c/go.mod h1:vdxJOQaD9MexhM0evO8XgV8m+dkiXxA9M/WkOgeB3P4= github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04= @@ -13,6 +46,9 @@ github.com/bytedance/sonic v1.12.1/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKz github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -20,15 +56,33 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7DV0IFes= +github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creeper12356/langchaingo v0.0.0-20240924140949-8ffd9dd88de7 h1:V0dtUt1K32yfVhsG4l/VOWieOntUMt9J9rouDXaiYdk= +github.com/creeper12356/langchaingo v0.0.0-20240924140949-8ffd9dd88de7/go.mod h1:vjzUeUsmulZ7Hwq0Ju3Ez+ZmM5aw0dA7K7K2u7wooIw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= +github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -47,6 +101,16 @@ github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GM github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-ego/gse v0.80.3 h1:YNFkjMhlhQnUeuoFcUEd1ivh6SOB764rT8GDsEbDiEg= github.com/go-ego/gse v0.80.3/go.mod h1:Gt3A9Ry1Eso2Kza4MRaiZ7f2DTAvActmETY46Lxg0gU= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -59,12 +123,18 @@ github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6 github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= +github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -74,9 +144,17 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.2 h1:WRkNAv2uoa03QNIc1A6u4O7DAGMUVoopZhkiXWA2V1o= github.com/gorilla/context v1.1.2/go.mod h1:KDPwT9i/MeWHiLl90fuTgrt4/wPcv75vFAZLaOOcbxM= @@ -92,6 +170,10 @@ github.com/gwatts/gin-adapter v1.0.0 h1:TsmmhYTR79/RMTsfYJ2IQvI1F5KZ3ZFJxuQSYEOp github.com/gwatts/gin-adapter v1.0.0/go.mod h1:44AEV+938HsS0mjfXtBDCUZS9vONlF2gwvh8wu4sRYc= github.com/hibiken/asynq v0.24.1 h1:+5iIEAyA9K/lcSPvx3qoPtsKJeKI5u9aOIvUmSsazEw= github.com/hibiken/asynq v0.24.1/go.mod h1:u5qVeSbrnfT+vtG5Mq8ZPzQu/BmCKMHvTGb91uy9Tts= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -108,6 +190,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -123,29 +207,57 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ= github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pgvector/pgvector-go v0.1.1 h1:kqJigGctFnlWvskUiYIvJRNwUtQl/aMSUZVs0YWQe+g= github.com/pgvector/pgvector-go v0.1.1/go.mod h1:wLJgD/ODkdtd2LJK4l6evHXTuG+8PxymYAVomKHOWac= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU= github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= @@ -154,11 +266,16 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/sashabaranov/go-openai v1.26.0 h1:upM565hxdqvCxNzuAcEBZ1XsfGehH0/9kgk9rFVpDxQ= -github.com/sashabaranov/go-openai v1.26.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= @@ -175,17 +292,57 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE= -github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4= +github.com/testcontainers/testcontainers-go v0.31.0 h1:W0VwIhcEVhRflwL9as3dhY6jXjVCA27AkmbnZ+UTh3U= +github.com/testcontainers/testcontainers-go v0.31.0/go.mod h1:D2lAoA0zUFiSY+eAflqK5mcUx/A5hrrORaEQrd0SefI= +github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 h1:isAwFS3KNKRbJMbWv+wolWqOFUECmjYZ+sIRZCIBc/E= +github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0/go.mod h1:ZNYY8vumNCEG9YI59A9d6/YaMY49uwRhmeU563EzFGw= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= github.com/vcaesar/cedar v0.20.2 h1:TDx7AdZhilKcfE1WvdToTJf5VrC/FXcUOW+KY1upLZ4= github.com/vcaesar/cedar v0.20.2/go.mod h1:lyuGvALuZZDPNXwpzv/9LyxW+8Y6faN7zauFezNsnik= github.com/vcaesar/tt v0.20.1 h1:D/jUeeVCNbq3ad8M7hhtB3J9x5RZ6I1n1eZ0BJp7M+4= github.com/vcaesar/tt v0.20.1/go.mod h1:cH2+AwGAJm19Wa6xvEa+0r+sXDJBT0QgNQey6mwqLeU= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 h1:Ss6D3hLXTM0KobyBYEAygXzFfGcjnmfEJOBgSbemCtg= +go.starlark.net v0.0.0-20230302034142-4b1e35fe2254/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= @@ -194,14 +351,20 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= @@ -228,10 +391,22 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= +google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -247,6 +422,8 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AW gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -254,6 +431,8 @@ gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= @@ -263,3 +442,5 @@ modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 4f40a85674f50710ab74500acb7777e2d36b77a8 Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 00:35:24 +0800 Subject: [PATCH 09/14] fix(llm): update docker-compose.yml and env config --- docker-compose.yml | 18 ++---------------- rpc/llm.go | 12 ++++++------ 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 652daec..ae91d67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: postgres: - image: postgres:latest + image: pgvector/pgvector:pg16 restart: always environment: POSTGRES_USER: jcourse @@ -13,19 +13,6 @@ services: ports: - 5432:5432 - vectordb: - image: pgvector/pgvector:pg16 - restart: always - environment: - POSTGRES_USER: jcourse - POSTGRES_PASSWORD: jcourse - POSTGRES_DB: jcourse - volumes: - - vector_data:/var/lib/postgresql/data - ports: - - 5433:5432 - - pgAdmin: container_name: pgAdmin # restart: unless-stopped @@ -47,5 +34,4 @@ services: - 6379:6379 volumes: postgres_data: - pgadmin: - vector_data: \ No newline at end of file + pgadmin: \ No newline at end of file diff --git a/rpc/llm.go b/rpc/llm.go index 7c0fe43..d37c7d2 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -3,7 +3,7 @@ package rpc import ( "context" "fmt" - "os" + "jcourse_go/util" // "github.com/sashabaranov/go-openai" "github.com/tmc/langchaingo/embeddings" @@ -33,11 +33,11 @@ import ( func getVectorDBConnUrl() string { return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", - os.Getenv("VECTORDB_USER"), - os.Getenv("VECTORDB_PASSWORD"), - os.Getenv("VECTORDB_HOST"), - os.Getenv("VECTORDB_PORT"), - os.Getenv("VECTORDB_DBNAME"), + util.GetPostgresUser(), + util.GetPostgresPassword(), + util.GetPostgresHost(), + util.GetPostgresPort(), + util.GetPostgresPassword(), ) } func OpenVectorStoreConn() (*pgvector.Store, error) { From ee4d3015e6aada383289f0665408597e6584797f Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 00:37:58 +0800 Subject: [PATCH 10/14] fix(llm): remove fmt.Println statements --- handler/llm.go | 5 +---- rpc/llm.go | 4 ++-- service/llm.go | 23 +++++++++++------------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/handler/llm.go b/handler/llm.go index 6593b00..77ca8ca 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -1,7 +1,6 @@ package handler import ( - "fmt" "jcourse_go/model/dto" "jcourse_go/service" "net/http" @@ -18,7 +17,6 @@ func OptCourseReviewHandler(c *gin.Context) { response, err := service.OptCourseReview(request.CourseName, request.ReviewContent) if err != nil { - fmt.Println(err) c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return } @@ -34,7 +32,6 @@ func GetCourseSummaryHandler(c *gin.Context) { response, err := service.GetCourseSummary(c, request.CourseID) if err != nil { - fmt.Println(err) c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return } @@ -50,7 +47,7 @@ func VectorizeCourseHandler(c *gin.Context) { err := service.VectorizeCourse(c, request.CourseID) if err != nil { - fmt.Println(err) + c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return } diff --git a/rpc/llm.go b/rpc/llm.go index d37c7d2..9d1230c 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -43,13 +43,13 @@ func getVectorDBConnUrl() string { func OpenVectorStoreConn() (*pgvector.Store, error) { llm, err := openai.New() if err != nil { - fmt.Println(err) + return nil, err } embedder, err := embeddings.NewEmbedder(llm) if err != nil { - fmt.Println(err) + return nil, err } diff --git a/service/llm.go b/service/llm.go index c6fba1c..555a50d 100644 --- a/service/llm.go +++ b/service/llm.go @@ -3,7 +3,6 @@ package service import ( "context" "encoding/json" - "fmt" "jcourse_go/constant" "jcourse_go/dal" "jcourse_go/model/converter" @@ -28,7 +27,7 @@ import ( func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseReviewResponse, error) { llm, err := openai.New() if err != nil { - fmt.Println(err) + return dto.OptCourseReviewResponse{}, err } inputJson, _ := json.Marshal(map[string]string{ @@ -47,7 +46,7 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi ) if err != nil { - fmt.Println(err) + return dto.OptCourseReviewResponse{}, err } var response dto.OptCourseReviewResponse @@ -89,13 +88,13 @@ func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummar reviews, err := GetReviewList(ctx, filter) if err != nil { - fmt.Println(err) + return nil, err } llm, err := openai.New() if err != nil { - fmt.Println(err) + return nil, err } @@ -118,7 +117,7 @@ func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummar ) if err != nil { - fmt.Println(err) + return nil, err } @@ -152,7 +151,7 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { reviews, err := GetReviewList(ctx, filter) if err != nil { - fmt.Println(err) + return err } @@ -165,7 +164,7 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { vectorStore, err := rpc.OpenVectorStoreConn() if err != nil { - fmt.Println(err) + return err } @@ -184,7 +183,7 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { ) if err != nil { - fmt.Println(err) + return err } @@ -202,19 +201,19 @@ func GetMatchCourses(ctx context.Context, description string) ([]model.CourseSum vectorStore, err := rpc.OpenVectorStoreConn() if err != nil { - fmt.Println(err) + return nil, err } docs, err := vectorStore.SimilaritySearch(context.Background(), description, 2) if err != nil { - fmt.Println(err) + return nil, err } err = vectorStore.Close() if err != nil { - fmt.Println(err) + return nil, err } From c064534dfe78c69a8230ced8579aee05c1ea5a18 Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 00:44:03 +0800 Subject: [PATCH 11/14] chore(llm): make lint --- go.mod | 1 + go.sum | 8 ++++---- handler/llm.go | 3 ++- router.go | 2 +- rpc/llm.go | 1 + service/llm.go | 3 ++- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index b94053d..9d96ea8 100644 --- a/go.mod +++ b/go.mod @@ -78,6 +78,7 @@ require ( golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.25.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 696abb4..d514f8d 100644 --- a/go.sum +++ b/go.sum @@ -355,8 +355,8 @@ golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xpp golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -391,8 +391,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/handler/llm.go b/handler/llm.go index 77ca8ca..bb1ada7 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -1,9 +1,10 @@ package handler import ( + "net/http" + "jcourse_go/model/dto" "jcourse_go/service" - "net/http" "github.com/gin-gonic/gin" ) diff --git a/router.go b/router.go index 08ece5a..27bc673 100644 --- a/router.go +++ b/router.go @@ -79,5 +79,5 @@ func registerRouter(r *gin.Engine) { llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) llmGroup.GET("/course/summary/:courseID", handler.GetCourseSummaryHandler) llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseHandler) - llmGroup.GET("/match", handler.GetMatchCoursesHandler) + llmGroup.GET("/course/match", handler.GetMatchCoursesHandler) } diff --git a/rpc/llm.go b/rpc/llm.go index 9d1230c..c25a5e5 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -3,6 +3,7 @@ package rpc import ( "context" "fmt" + "jcourse_go/util" // "github.com/sashabaranov/go-openai" diff --git a/service/llm.go b/service/llm.go index 555a50d..025e9ea 100644 --- a/service/llm.go +++ b/service/llm.go @@ -3,6 +3,8 @@ package service import ( "context" "encoding/json" + "strings" + "jcourse_go/constant" "jcourse_go/dal" "jcourse_go/model/converter" @@ -10,7 +12,6 @@ import ( "jcourse_go/model/model" "jcourse_go/repository" "jcourse_go/rpc" - "strings" "github.com/tmc/langchaingo/llms" "github.com/tmc/langchaingo/llms/openai" From 3e458393ed866bc69c38806b05d29e57f3797a14 Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 00:46:46 +0800 Subject: [PATCH 12/14] chore(llm): delete pgAdmin docker compose service --- docker-compose.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ae91d67..60cf18e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,20 +13,6 @@ services: ports: - 5432:5432 - pgAdmin: - container_name: pgAdmin - # restart: unless-stopped - image: dpage/pgadmin4:latest - environment: - PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org} - PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} - PGADMIN_LISTEN_PORT: 5050 - volumes: - - pgadmin:/root/.pgadmin - ports: - - "5050:5050" - - redis: image: redis:latest restart: always From b78d22fbf2f17c7aa434ee6bde2aa64b26fb19aa Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 23:03:47 +0800 Subject: [PATCH 13/14] fix(llm): replace context with input args --- handler/llm.go | 2 +- rpc/llm.go | 2 +- service/llm.go | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/handler/llm.go b/handler/llm.go index bb1ada7..a222eeb 100644 --- a/handler/llm.go +++ b/handler/llm.go @@ -16,7 +16,7 @@ func OptCourseReviewHandler(c *gin.Context) { return } - response, err := service.OptCourseReview(request.CourseName, request.ReviewContent) + response, err := service.OptCourseReview(c, request.CourseName, request.ReviewContent) if err != nil { c.JSON(http.StatusInternalServerError, dto.BaseResponse{Message: "内部错误。"}) return diff --git a/rpc/llm.go b/rpc/llm.go index c25a5e5..7c9c9f1 100644 --- a/rpc/llm.go +++ b/rpc/llm.go @@ -41,7 +41,7 @@ func getVectorDBConnUrl() string { util.GetPostgresPassword(), ) } -func OpenVectorStoreConn() (*pgvector.Store, error) { +func OpenVectorStoreConn(ctx context.Context) (*pgvector.Store, error) { llm, err := openai.New() if err != nil { diff --git a/service/llm.go b/service/llm.go index 025e9ea..3ee3cbe 100644 --- a/service/llm.go +++ b/service/llm.go @@ -25,7 +25,7 @@ import ( // 函数返回值包含两个字段： // Suggestion为修改建议， // Result为根据修改建议给出的一种修改结果。 -func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseReviewResponse, error) { +func OptCourseReview(ctx context.Context, courseName string, reviewContent string) (dto.OptCourseReviewResponse, error) { llm, err := openai.New() if err != nil { @@ -42,7 +42,7 @@ func OptCourseReview(courseName string, reviewContent string) (dto.OptCourseRevi } completion, err := llm.GenerateContent( - context.Background(), + ctx, content, ) @@ -113,7 +113,7 @@ func GetCourseSummary(ctx context.Context, courseID int64) (*dto.GetCourseSummar } completion, err := llm.GenerateContent( - context.Background(), + ctx, content, ) @@ -162,7 +162,7 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { comments = append(comments, review.Comment) } - vectorStore, err := rpc.OpenVectorStoreConn() + vectorStore, err := rpc.OpenVectorStoreConn(ctx) if err != nil { @@ -178,7 +178,7 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { } _, err = vectorStore.AddDocuments( - context.Background(), + ctx, []schema.Document{doc}, vectorstores.WithReplacement(true), ) @@ -199,14 +199,14 @@ func VectorizeCourse(ctx context.Context, courseID int64) error { // 输出的课程列表数量为2，后续可以修改。 func GetMatchCourses(ctx context.Context, description string) ([]model.CourseSummary, error) { - vectorStore, err := rpc.OpenVectorStoreConn() + vectorStore, err := rpc.OpenVectorStoreConn(ctx) if err != nil { return nil, err } - docs, err := vectorStore.SimilaritySearch(context.Background(), description, 2) + docs, err := vectorStore.SimilaritySearch(ctx, description, 2) if err != nil { return nil, err From 7f533db46baca77d3c1a309ea307b3c73253279f Mon Sep 17 00:00:00 2001 From: creeper Date: Thu, 26 Sep 2024 23:09:27 +0800 Subject: [PATCH 14/14] fix(llm): only enable vectorize api in DEBUG mode --- router.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/router.go b/router.go index 27bc673..5e086d4 100644 --- a/router.go +++ b/router.go @@ -78,6 +78,9 @@ func registerRouter(r *gin.Engine) { llmGroup := needAuthGroup.Group(("/llm")) llmGroup.GET("/review/opt", handler.OptCourseReviewHandler) llmGroup.GET("/course/summary/:courseID", handler.GetCourseSummaryHandler) - llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseHandler) llmGroup.GET("/course/match", handler.GetMatchCoursesHandler) + + if util.IsDebug() { + llmGroup.GET("/vectorize/:courseID", handler.VectorizeCourseHandler) + } }