From 568d1d1dabf334f8921101585a806534efc6760f Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 18:42:45 +0700 Subject: [PATCH] chore: changeset feat: add cli reindex and delete index feat: create reindex guideline chore: update courseDoc fix: invalid command chore: update index and scraper --- .changeset/red-baboons-brush.md | 5 ++ apps/opensearch-cli/README.md | 39 +++++++++ apps/opensearch-cli/cli/app.go | 87 +++++++++++++++++++ apps/opensearch-cli/docs/reindex.md | 5 ++ .../index/beta/course-index.json | 8 +- .../index/dev/course-index.json | 8 +- .../index/prod/course-index.json | 8 +- .../queue-consumer/queue-consumer.service.ts | 8 ++ 8 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 .changeset/red-baboons-brush.md create mode 100644 apps/opensearch-cli/docs/reindex.md diff --git a/.changeset/red-baboons-brush.md b/.changeset/red-baboons-brush.md new file mode 100644 index 000000000..6736cf89f --- /dev/null +++ b/.changeset/red-baboons-brush.md @@ -0,0 +1,5 @@ +--- +'api': minor +--- + +fix: implement s/u filter in search diff --git a/apps/opensearch-cli/README.md b/apps/opensearch-cli/README.md index 435bf1c59..0787d005a 100644 --- a/apps/opensearch-cli/README.md +++ b/apps/opensearch-cli/README.md @@ -26,3 +26,42 @@ go run ./cli/app.go index create --index-name= --filename= ```shell go run ./cli/app.go index create --index-name=course-dev-1 --filename=dev/course-index.json ``` + +# Reindex + +```shell +go run ./cli/app.go index reindex --index-name= --dest= --debug= +``` + +| name | description | +| ---------- | ---------------------------------- | +| index name | name of index | +| dest | name of destination index | +| debug | enable debug mode (`true`/`false`) | + +## Example + +### Reindex from course-dev-1 to course-dev-2 + +```shell +go run ./cli/app.go index reindex --index-name=course-dev-1 --dest=course-dev-2 +``` + +# Delete Index + +```shell +go run ./cli/app.go index reindex --index-name= --dest= --debug= +``` + +| name | description | +|------------|------------------------------------| +| index name | name of index | +| debug | enable debug mode (`true`/`false`) | + +## Example + +### Delete index course-dev-1 + +```shell +go run ./cli/app.go index delete --index-name=course-dev-1 +``` diff --git a/apps/opensearch-cli/cli/app.go b/apps/opensearch-cli/cli/app.go index 1447e4810..95a7e4553 100644 --- a/apps/opensearch-cli/cli/app.go +++ b/apps/opensearch-cli/cli/app.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "github.com/opensearch-project/opensearch-go/v2" @@ -18,6 +19,7 @@ import ( var ( fileName string indexName string + destName string isDebug bool indexCmd *flag.FlagSet ) @@ -28,6 +30,7 @@ func init() { indexCmd = flag.NewFlagSet("index", flag.ExitOnError) indexCmd.StringVar(&indexName, "index-name", "", "Index name") + indexCmd.StringVar(&destName, "dest", "", "Destination index name") indexCmd.StringVar(&fileName, "filename", "", "Index filename") indexCmd.BoolVar(&isDebug, "debug", false, "Enable the debug log") flag.Parse() @@ -74,6 +77,26 @@ func main() { fmt.Println(strings.Repeat("_", 65)) fmt.Printf("successfully create the index, %s \n", indexName) + case "reindex": + if err := handleReindex(client); err != nil { + logger.Fatal( + "Error while creating the index", + zap.Error(err), + ) + } + + fmt.Println(strings.Repeat("_", 65)) + fmt.Printf("successfully reindex from %s to %s \n", indexName, destName) + case "delete": + if err := handleDeleteIndex(client); err != nil { + logger.Fatal( + "Error while creating the index", + zap.Error(err), + ) + } + + fmt.Println(strings.Repeat("_", 65)) + fmt.Printf("successfully delete the index, %s \n", indexName) default: logger.Fatal("Invalid input (invalid command)") } @@ -146,3 +169,67 @@ func handleCreateIndex(client *opensearch.Client) error { return nil } + +type ReIndexBody struct { + Source struct { + Index string `json:"index"` + } `json:"source"` + + Dest struct { + Index string `json:"index"` + } `json:"dest"` +} + +func handleReindex(client *opensearch.Client) error { + if err := indexCmd.Parse(os.Args[3:]); err != nil { + return err + } + + req, err := json.Marshal(ReIndexBody{ + Source: struct { + Index string `json:"index"` + }{Index: indexName}, + + Dest: struct { + Index string `json:"index"` + }{Index: destName}, + }) + + if err != nil { + return err + } + + res, err := client.Reindex( + bytes.NewReader(req), + ) + + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil +} + +func handleDeleteIndex(client *opensearch.Client) error { + if err := indexCmd.Parse(os.Args[3:]); err != nil { + return err + } + + res, err := client.Indices.Delete( + []string{indexName}, + ) + + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil +} diff --git a/apps/opensearch-cli/docs/reindex.md b/apps/opensearch-cli/docs/reindex.md new file mode 100644 index 000000000..4d8d73f13 --- /dev/null +++ b/apps/opensearch-cli/docs/reindex.md @@ -0,0 +1,5 @@ +# Reindex Guideline + +1. Create new index (destination index) by using command `go run ./cli/app.go index create --index-name= --filename=` +2. Run reindex command `go run ./cli/app.go index reindex --index-name= --dest=` +3. Delete the old one `go run ./cli/app.go index delete --index-name=` diff --git a/apps/opensearch-cli/index/beta/course-index.json b/apps/opensearch-cli/index/beta/course-index.json index b319ac5d3..4f3586738 100644 --- a/apps/opensearch-cli/index/beta/course-index.json +++ b/apps/opensearch-cli/index/beta/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-beta": {} + "cgr-course-beta": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/opensearch-cli/index/dev/course-index.json b/apps/opensearch-cli/index/dev/course-index.json index 25851cb4b..5deefe74f 100644 --- a/apps/opensearch-cli/index/dev/course-index.json +++ b/apps/opensearch-cli/index/dev/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-dev": {} + "cgr-course-dev": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/opensearch-cli/index/prod/course-index.json b/apps/opensearch-cli/index/prod/course-index.json index 1fa3ec39b..a9a0bb661 100644 --- a/apps/opensearch-cli/index/prod/course-index.json +++ b/apps/opensearch-cli/index/prod/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-prod": {} + "cgr-course-prod": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts index eaf81721d..159393621 100644 --- a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts +++ b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts @@ -26,6 +26,7 @@ interface CourseDoc { studyProgram: StudyProgram semester: Semester academicYear: string + creditHours: string } @Processor({ @@ -76,6 +77,13 @@ export class QueueConsumerService { genEdType: course.genEdType, semester: course.semester, studyProgram: course.studyProgram, + creditHours: course.creditHours, + suggestions: [ + { + input: [course.courseNo, course.abbrName, course.courseNameEn, course.courseNameTh], + weight: 20, + }, + ], } as CourseDoc, doc_as_upsert: true, },