diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e336e8b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +*.swp +*.swo +db.sql +words +migrate/migrate +words_image.tar +loader_image.tar +data/*.bz2 +commands.md +cover.out +testdata/* +backups/* +.vscode +.git +.cache diff --git a/.gitignore b/.gitignore index fa76a7c..5381cad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,14 @@ secret.go *.swo db.sql words +migrate/migrate words_image.tar +loader_image.tar data/*.csv data/*.bz2 commands.md cover.out testdata/e2e_corrected.json backups/* + +.vscode/ diff --git a/Dockerfile b/Dockerfile index ef5398a..9cad5c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ FROM golang:alpine AS builder RUN apk update && apk add --no-cache git gcc g++ ca-certificates apache2-utils WORKDIR /go/src/words COPY *.go ./ -Run go get -d -v -tags netgo -installsuffix netgo +RUN go get -d -v -tags netgo -installsuffix netgo # netgo and ldflags makes sure that dns resolver and binary are statically # linked giving the ability for smaller images. RUN go build -tags netgo -installsuffix netgo -ldflags '-extldflags "-static"' -o /go/bin/words @@ -28,6 +28,5 @@ FROM scratch # solution? COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /go/bin/words /go/bin/words -COPY data/*.csv data/ # TODO: This should be moved to the CMD starting it up. ENTRYPOINT ["/go/bin/words", "--db_path=/words-vol/db/db.sql"] diff --git a/README.md b/README.md index b6df726..ef70e25 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,23 @@ repository. It should live in data/links.csv and data/sentences.csv It can be downloaded from here: https://tatoeba.org/eng/downloads + +## QuickStart: +1. Create a telegram bot using @BotFather if you don't have one yet +2. Create secret.go in the root folder with the following content +```go + +package main + +const BotToken = "TOKEN PROVIDED BY BOT FATHER" +``` +3. Download links and senteces from https://tatoeba.org/eng/downloads +4. Put fetched csv data under `./data` +5. Run using one of the following: +- using Go: `go build && ./words` +- using Docker: +```bash +sudo docker volume create words-vol +sudo docker build -t words . +sudo docker run --rm --name words-app --mount source=words-vol,target=/words-vol/db/ words +``` diff --git a/actions.go b/actions.go index abcf97b..6ed8e52 100644 --- a/actions.go +++ b/actions.go @@ -17,24 +17,7 @@ // users can take. package main -import ( - "database/sql" - "fmt" - "log" - "math/rand" - "sort" - "strings" -) - -// Questions: -// * Should actions be exclusive of one another? If one is executed, the rest are skipped? -// * What about calling if !Match in each action to make sure preconditions hold? NO. Difficult chaining to reuse. -type Action interface { - // Perform the action. - Perform(*Update) ([]Action, error) - // Whether this action should be performed as a result of this update. - Match(*Update) bool -} +import "fmt" // TODO: I am not sure if this is the best decision to bundle all up together. // All objects needed to perform actions. @@ -45,314 +28,6 @@ type Clients struct { Settings *SettingsConfig } -// Actions that always can be executed independent of other things. -func BaseActions(c *Clients) []Action { - return []Action{ - &DontKnowAction{c, ""}, - &LearnAction{c}, - &DeleteWordAction{c}, - &CatchAllAction{c, ""}, - } -} - -// actions shared by all commands: -// b:Don't know action (reset) - to work with all messages, no further -// practice to display. -// b:Learn - should always be possible to add another word to the ones learning - -// THese are more for RegularActions or Default actions -func DefaultActions(c *Clients) []Action { - return []Action{ - &StartAction{c}, - &SettingsAction{c}, - &PracticeAction{c}, - &AddCommandAction{c}, - &DefineWordAction{c}, - // /start - // /settings - // /practice - // word - /// catch the rest and show help? - } -} - -// Practice Actions: -// b:Know & display another (If exited practice mode, or word from callback != -// word from action - do nothing) -// b:Don't know & display another (same as b:Know) -// /stop -// /settings -/// -// (may or may not include /start) - -// func NewPracticeActions(c *Clients, asking string) []Action { -// return []Action{ -// &StartAction{}, -// } -// } - -// PracticeKnowAction{asking string} -// PracticeKnowAction.Perform{ ... return NewPracticeState(a.asking) } - -type StartAction struct { - *Clients -} - -// CallbackAction as a special action to handle all callbacks? - -func (a *StartAction) Perform(u *Update) ([]Action, error) { - // It might be beneficial not to reset actions here. - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(u.Message.Chat.Id, - "Welcome to the language bot. Still in development. No instructions "+ - "so far. "+ - "All sentences and translations are from Tatoeba's (https://tatoeba.org) "+ - "dataset, released under a CC-BY 2.0 FR.") -} - -func (*StartAction) Match(u *Update) bool { - // FIXME: This looks refactorable! - if u.Message == nil { - return false - } - return u.Message.Text == "/start" -} - -type StopAction struct { - *Clients - Message string -} - -func (a *StopAction) Perform(u *Update) ([]Action, error) { - msg := a.Message - if msg == "" { - msg = "Stopped." - } - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(u.Message.Chat.Id, - msg) -} - -func (a *StopAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/stop" -} - -// An action to be executed if no other action was matched. -type CatchAllAction struct { - *Clients - Message string -} - -func (a *CatchAllAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - msg := a.Message - if msg == "" { - switch { - case u.Message != nil: - msg += fmt.Sprintf("Couldn't process your message %q", u.Message.Text) - case u.CallbackQuery != nil: - msg += fmt.Sprintf("INTERNAL: Due to restart or a bug couldn't process callback query %v", u.CallbackQuery) - default: - msg += fmt.Sprintf("INTERNAL: Cannot handle update %v", u) - } - } - return nil, a.Telegram.SendTextMessage(chatId, msg) -} - -func (a *CatchAllAction) Match(u *Update) bool { - return true -} - -type CatchAllMessagesAction struct{ CatchAllAction } - -func (a *CatchAllMessagesAction) Match(u *Update) bool { - return u.Message != nil -} - -// FIXME: Should they be "inherited" from Clients? (I think that makes sense) -type SettingsAction struct { - *Clients -} - -func (a *SettingsAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/settings" -} - -func (a *SettingsAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - s, err := a.Settings.Get(chatId) - if err != nil { - return nil, err - } - var ls []string - for l, v := range s.TranslationLanguages { - if v { - ls = append(ls, fmt.Sprintf("%q", l)) - } - } - sort.Strings(ls) - msg := fmt.Sprintf(` -Current settings: - -Input language: %q -Input language in ISO 639-3: %q -Translation languages in ISO 639-3: %s - -Choose setting which you want to modify. -(The choice might improve in the future.) -`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ",")) - return []Action{ - // Note that stop should be handled before input language is! - &StopAction{a.Clients, "Exited settings"}, - &InputLanguageButton{a.Clients}, - &PracticeAction{a.Clients}, - &CatchAllMessagesAction{CatchAllAction{ - a.Clients, "type /stop to exit settings"}}, - }, a.Telegram.SendMessage(&MessageReply{ - ChatId: chatId, - Text: msg, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - &InlineKeyboard{ - Text: "Input Language", - CallbackData: CallbackInfo{ - Action: ChangeSettingAction, - Setting: "InputLanguage", - }.String(), - }, - }}, - }, - }) -} - -type InputLanguageButton struct { - *Clients -} - -func (b *InputLanguageButton) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - var ls []string - for l, _ := range SupportedInputLanguages { - ls = append(ls, fmt.Sprintf("%q", l)) - } - sort.Strings(ls) - return []Action{ - &StopAction{b.Clients, "Exited settings"}, - &PickLanguageAction{b.Clients}, - }, b.Telegram.SendTextMessage( - chatId, - fmt.Sprintf("Enter input language of your choice. Supported are %s", - strings.Join(ls, ","))) -} - -func (b *InputLanguageButton) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Setting == "InputLanguage" -} - -type PickLanguageAction struct { - *Clients -} - -func (a *PickLanguageAction) Perform(u *Update) ([]Action, error) { - m := u.Message - chatId := m.Chat.Id - set, ok := SupportedInputLanguages[m.Text] - if !ok { - return nil, a.Telegram.SendTextMessage(chatId, "Unsupported language. Try again.") - } - a.Settings.Set(chatId, &set) - sa := SettingsAction{a.Clients} - return sa.Perform(u) -} - -func (a *PickLanguageAction) Match(u *Update) bool { - return u.Message != nil -} - -type PracticeAction struct { - *Clients -} - -func (a *PracticeAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - switch UsePractice { - case PracticeWordSpelling: - // FIXME: This is not tested!!! - d, err := a.Repetitions.Repeat(chatId) - if err == sql.ErrNoRows { - // FIXME: This might be refactorable - return DefaultActions(a.Clients), - a.Telegram.SendTextMessage(chatId, "No more rows to practice; exiting practice mode.") - } - if err != nil { - return nil, fmt.Errorf("retrieving word for repetition: %w", err) - } - return []Action{ - &StopAction{a.Clients, "Stopped practice"}, - &SettingsAction{a.Clients}, - &AnswerAction{a.Clients, d}, - // FIXME: Obfuscate definition before sending it to the user!!! - }, a.Telegram.SendTextMessage(chatId, d) - case PracticeKnowledge: - w, err := a.Repetitions.RepeatWord(chatId) - if err == sql.ErrNoRows { - return DefaultActions(a.Clients), - a.Telegram.SendTextMessage(chatId, "No more rows to practice; exiting practice mode.") - } - if err != nil { - return nil, fmt.Errorf("retrieving word for repetition: %w", err) - } - ka := &KnowAction{a.Clients, w} - dka := &DontKnowAction{a.Clients, w} - return []Action{ - &StopAction{a.Clients, "Stopped practice"}, - &SettingsAction{a.Clients}, - ka, - dka, - &CatchAllMessagesAction{CatchAllAction{ - a.Clients, "type /stop to exit practice mode"}}, - }, a.Telegram.SendMessage(&MessageReply{ - ChatId: chatId, - Text: w, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - ka.AsKeyboard(), - dka.AsKeyboard(), - }}, - }, - }) - default: - return nil, fmt.Errorf("INTERNAL: Unimplemented practice type: %v", UsePractice) - } -} - -func (a *PracticeAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/practice" -} - // TODO: Can I not extract word from the message? m.Text? func flipWordCard(c *Clients, word string, m *Message, ks []*InlineKeyboard) error { // TODO: It isn't always neccessary to retrieve defitnion when this @@ -382,317 +57,3 @@ func flipWordCard(c *Clients, word string, m *Message, ks []*InlineKeyboard) err } return nil } - -type KnowAction struct { - *Clients - Word string -} - -func (a *KnowAction) Perform(u *Update) ([]Action, error) { - defer a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, "") - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.AnswerKnow(chatId, word); err != nil { - return nil, err - } - dka := &DontKnowAction{a.Clients, a.Word} - if err := flipWordCard(a.Clients, word, u.CallbackQuery.Message, []*InlineKeyboard{dka.AsKeyboard()}); err != nil { - return nil, err - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *KnowAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == PracticeKnowAction && info.Word == a.Word -} - -func (a *KnowAction) AsKeyboard() *InlineKeyboard { - return &InlineKeyboard{ - Text: "Know", - CallbackData: CallbackInfo{ - Action: PracticeKnowAction, - Word: a.Word, - }.String(), - } -} - -type DontKnowAction struct { - *Clients - Word string -} - -func (a *DontKnowAction) Perform(u *Update) ([]Action, error) { - defer a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, "Reset progress") - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.AnswerDontKnow(chatId, word); err != nil { - return nil, err - } - if err := flipWordCard(a.Clients, word, u.CallbackQuery.Message, nil); err != nil { - return nil, err - } - // Clicking Don't know on the previous messages shouldn't result in - // practice being continued. - if word != a.Word { - return nil, nil - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *DontKnowAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == PracticeDontKnowAction -} - -func (a *DontKnowAction) AsKeyboard() *InlineKeyboard { - return &InlineKeyboard{ - Text: "Don't know", - CallbackData: CallbackInfo{ - Action: PracticeDontKnowAction, - Word: a.Word, - }.String(), - } -} - -type AnswerAction struct { - *Clients - Question string -} - -func (a *AnswerAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - m := u.Message - correct, err := a.Repetitions.Answer(chatId, a.Question, m.Text) - if err != nil { - return nil, err - } - if correct != m.Text { - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Correct word: %q", correct)) - } - // correct == m.Text - congrats := [...]string{"Good job!", "Well done!", "Awesome!", "Keep it up!", "Excellent!", "You've got it!", "Great!", "Terrific!", "Ваще красава!11"} - if err := a.Telegram.SendTextMessage(chatId, congrats[rand.Int()%len(congrats)]); err != nil { - return nil, err - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *AnswerAction) Match(u *Update) bool { - return u.Message != nil -} - -type AddCommandAction struct { - *Clients -} - -func (a *AddCommandAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - return []Action{ - &StopAction{a.Clients, "Cancelled addition"}, - &SettingsAction{a.Clients}, - &AddWordAction{a.Clients}, - }, a.Telegram.SendTextMessage(chatId, ` -Enter the card you want to add in the format: - - -`) -} - -func (a *AddCommandAction) Match(u *Update) bool { - return u.Message != nil && u.Message.Text == "/add" -} - -type AddWordAction struct { - *Clients -} - -func (a *AddWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - parts := strings.Split(u.Message.Text, "\n") - if len(parts) < 2 { - return nil, a.Telegram.SendTextMessage(chatId, "Wrong format") - } - front := parts[0] - back := strings.Join(parts[1:], "\n") - if err := a.Repetitions.Save(chatId, front, back); err != nil { - return nil, err - } - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Added %q for learning!", front)) -} - -func (a *AddWordAction) Match(u *Update) bool { - return u.Message != nil -} - -type DefineWordAction struct { - *Clients -} - -func (a *DefineWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - m := u.Message - if len(strings.Split(m.Text, " ")) > 1 { - return nil, a.Telegram.SendTextMessage(chatId, "INTERNAL ERROR: DefineWordAction should not have been called with many words. Is Match called correctly?") - } - - def, err := a.Repetitions.GetDefinition(m.Chat.Id, m.Text) - if err == nil { - dka := &DontKnowAction{a.Clients, m.Text} - k := dka.AsKeyboard() - k.Text = "Reset progress" - return nil, a.Telegram.SendMessage(&MessageReply{ - ChatId: m.Chat.Id, - Text: def, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{ - []*InlineKeyboard{k}, - }, - }, - }) - } - if err != sql.ErrNoRows { - log.Printf("ERROR: Repetitions(%d, %s): %v", m.Chat.Id, m.Text, err) - } - settings, err := a.Settings.Get(chatId) - if err != nil { - return nil, fmt.Errorf("get settings: %v", err) - } - ds, err := a.Definer.Define(m.Text, settings) - if err != nil { - // TODO: Might be good to post debug logs to the reply in the debug mode. - log.Printf("Error fetching the definition: %w", err) - // FIXME: Add search url to the reply? - return nil, a.Telegram.SendTextMessage(chatId, "Couldn't find definitions") - } - - for _, d := range ds { - if err := a.Telegram.SendMessage(&MessageReply{ - ChatId: m.Chat.Id, - Text: d, - ParseMode: "MarkdownV2", - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - &InlineKeyboard{ - Text: "Learn", - CallbackData: CallbackInfo{ - Action: SaveWordAction, - Word: m.Text, - }.String(), - }, - }}, - }, - }); err != nil { - return nil, err - } - } - return nil, nil -} - -func (a *DefineWordAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return len(strings.Split(u.Message.Text, " ")) == 1 -} - -type LearnAction struct { - *Clients -} - -func (a *LearnAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.Save(chatId, word, u.CallbackQuery.Message.Text); err != nil { - return nil, err - } - m := u.CallbackQuery.Message - r := &EditMessageText{ - ChatId: m.Chat.Id, - MessageId: m.Id, - ReplyMarkup: ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{ - []*InlineKeyboard{}, - }, - }, - } - var rm Message - if err := a.Telegram.Call("editMessageReplyMarkup", r, &rm); err != nil { - return nil, fmt.Errorf("editing message reply markup: %w", err) - } - msg := fmt.Sprintf("Saved %q for learning", word) - a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, msg) - return nil, nil -} - -func (a *LearnAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == SaveWordAction -} - -// DeleteWordAction stops learning of the word. -type DeleteWordAction struct { - *Clients -} - -func (a *DeleteWordAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return strings.HasPrefix(u.Message.Text, "/delete") -} - -func (a *DeleteWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - word := strings.TrimSpace(strings.TrimPrefix(u.Message.Text, "/delete")) - e, err := a.Repetitions.Exists(chatId, word) - if err != nil { - return nil, err - } - if !e { - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Word %q isn't saved for learning!", word)) - } - if err := a.Repetitions.Delete(chatId, word); err != nil { - return nil, err - } - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Deleted %q!", word)) -} diff --git a/callbacks.go b/callbacks.go new file mode 100644 index 0000000..d2d33da --- /dev/null +++ b/callbacks.go @@ -0,0 +1,165 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +type KnowCallback struct { + Word string +} + +func (KnowCallback) Call(s *State, q *CallbackQuery) error { + defer s.Telegram.AnswerCallbackLog(q.Id, "") + chatID := q.Message.Chat.Id + word := CallbackInfoFromString(q.Data).Word + + // TODO: Need to handle 2 rapid taps to avoid saving it as known 2 times in a row. + if err := s.Repetitions.AnswerKnow(chatID, word); err != nil { + return err + } + + if err := flipWordCard(s.Clients, word, q.Message, []*InlineKeyboard{DontKnowCallback{word, false}.AsInlineKeyboard()}); err != nil { + return err + } + return practiceReply(s, chatID) +} + +func (KnowCallback) Match(s *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == PracticeKnowAction +} + +func (k KnowCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Know", + CallbackData: CallbackInfo{ + Action: PracticeKnowAction, + Word: k.Word, + }.String(), + } +} + +type DontKnowCallback struct { + Word string + // If true when clicking another practice card will be shown. + Practice bool +} + +func (DontKnowCallback) Call(s *State, q *CallbackQuery) error { + defer s.Telegram.AnswerCallbackLog(q.Id, "Reset progress") + info := CallbackInfoFromString(q.Data) + chatID := q.Message.Chat.Id + word := info.Word + + if err := s.Repetitions.AnswerDontKnow(chatID, word); err != nil { + return err + } + + if err := flipWordCard(s.Clients, word, q.Message, nil); err != nil { + return err + } + + if info.Action == PracticeDontKnowActionNoPractice { + return nil + } + return practiceReply(s, chatID) +} + +func (DontKnowCallback) Match(_ *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == PracticeDontKnowAction || info.Action == PracticeDontKnowActionNoPractice +} + +func (c DontKnowCallback) AsInlineKeyboard() *InlineKeyboard { + a := PracticeDontKnowActionNoPractice + if c.Practice { + a = PracticeDontKnowAction + } + return &InlineKeyboard{ + Text: "Don't know", + CallbackData: CallbackInfo{ + Action: a, + Word: c.Word, + }.String(), + } +} + +type ResetProgressCallback struct { + Word string +} + +// ResetProgress type is just a convenience placeholder to create inline keyboards. +// Logic will be handled by dont know callback. +func (ResetProgressCallback) Call(_ *State, _ *CallbackQuery) error { + return nil +} + +func (ResetProgressCallback) Match(_ *State, _ *CallbackQuery) bool { + return false +} + +func (c ResetProgressCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Reset progress", + CallbackData: CallbackInfo{ + Action: PracticeDontKnowActionNoPractice, + Word: c.Word, + }.String(), + } +} + +type LearnCallback struct { + Word string +} + +func (LearnCallback) Call(s *State, q *CallbackQuery) error { + // FIXME: Next 3 lines are very common. + chatID := q.Message.Chat.Id + word := CallbackInfoFromString(q.Data).Word + if err := s.Repetitions.Save(chatID, word, q.Message.Text); err != nil { + return err + } + m := q.Message + r := &EditMessageText{ + ChatId: m.Chat.Id, + MessageId: m.Id, + ReplyMarkup: ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{ + []*InlineKeyboard{}, + }, + }, + } + var rm Message + if err := s.Telegram.Call("editMessageReplyMarkup", r, &rm); err != nil { + return fmt.Errorf("editing message reply markup: %w", err) + } + msg := fmt.Sprintf("Saved %q for learning", word) + s.Telegram.AnswerCallbackLog(q.Id, msg) + return nil +} + +func (LearnCallback) Match(_ *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == SaveWordAction +} + +func (c LearnCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Learn", + CallbackData: CallbackInfo{ + Action: SaveWordAction, + Word: c.Word, + }.String(), + } +} diff --git a/commands.go b/commands.go index e1de0f7..11877b8 100644 --- a/commands.go +++ b/commands.go @@ -42,10 +42,6 @@ const ( const UsePractice = PracticeKnowledge var ( - SupportedInputLanguages map[string]Settings -) - -func init() { SupportedInputLanguages = map[string]Settings{ "Hungarian": Settings{ InputLanguage: "Hungarian", @@ -74,7 +70,15 @@ func init() { }, }, } -} + TimeZones = func() map[string]bool { + timeZones := make(map[string]bool) + for i := -12; i < 12; i++ { + timeZones[fmt.Sprintf("UTC%+d", i)] = true + } + timeZones["UTC"] = true + return timeZones + }() +) type CallbackAction int @@ -82,7 +86,7 @@ const ( SaveWordAction CallbackAction = iota PracticeKnowAction PracticeDontKnowAction - ChangeSettingAction + PracticeDontKnowActionNoPractice ) // Make sure all fields are Public, otherwise encoding will not work @@ -115,18 +119,13 @@ func (c CallbackInfo) String() string { type Commander struct { *Clients - // chat_id -> Available Actions - actions map[int64][]Action - // Actions that should be always available. - baseActions []Action + bot *Bot } type CommanderOptions struct { - useCache bool - dbPath string - stages []time.Duration - sentencesPath string - linksPath string // links between sentence and translations + useCache bool + dbPath string + stages []time.Duration } func escapeMarkdown(s string) string { @@ -167,10 +166,7 @@ func NewCommander(tm *Telegram, opts *CommanderOptions) (*Commander, error) { } // TODO: Can use errgroup if there is a need to paralelize. This is the // slowest step in initialization. - uf, err := NewUsageFetcher(UsageFetcherOptions{ - SentencesPath: opts.sentencesPath, - LinksPath: opts.linksPath, - }) + uf, err := NewUsageFetcher(opts.dbPath) if err != nil { return nil, fmt.Errorf("creating usage fetcher: %w", err) } @@ -202,48 +198,25 @@ func NewCommander(tm *Telegram, opts *CommanderOptions) (*Commander, error) { log.Printf("getMe: %s", string(raw)) return &Commander{ - Clients: c, - actions: make(map[int64][]Action), - baseActions: BaseActions(c), + Clients: c, + bot: &Bot{ + state: &State{c}, + command: make(map[int64]Command), + }, }, nil } -func (c *Commander) Actions(chatId int64) []Action { - a, ok := c.actions[chatId] - if !ok { - a = DefaultActions(c.Clients) - } - return append(a, c.baseActions...) -} - // Update processes the user's update and spit out output. // Should return an error only on unrecoverable errors due to which we cannot // continue execution. // TODO: Use answerCallbackQuery to notify client that callback was processed? func (c *Commander) Update(u *Update) error { - chatId, err := u.ChatId() + err := c.bot.Update(u) if err != nil { // Not sure what to do otherwise, but crashing isn't nice. - log.Printf("INTENAL ERROR: {%+v}.ChatId(): %v", u, err) - return nil + log.Printf("INTENAL ERROR for update %v: %v", u, err) } - for _, a := range c.Actions(chatId) { - if a.Match(u) { - na, err := a.Perform(u) - // FIXME: on error maybe some action changes are warranted? - if err != nil { - // FIXME: Don't return an error here!!! Display it. - return err - } - // FIXME seems annoying to keep track of current actions when on wron - // input the set of actions should never change. - if len(na) > 0 { - c.actions[chatId] = na - } - return nil - } - } - return fmt.Errorf("Did not process an update: %v!", u.CallbackQuery) + return nil } func (c *Commander) PollAndProcess() error { diff --git a/commandsV2.go b/commandsV2.go new file mode 100644 index 0000000..9f0275c --- /dev/null +++ b/commandsV2.go @@ -0,0 +1,654 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// commandsV2 is a replacement for action-based message processing. Everything +// inside here is responsible for tying database together with telegram +// interactions. +package main + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "sort" + "strings" +) + +type Callback interface { + Call(*State, *CallbackQuery) error + Match(*State, *CallbackQuery) bool + AsInlineKeyboard() *InlineKeyboard +} + +type State struct { + *Clients +} + +// TODO: +func (s *State) LoadCommand(chatID int64) (*SerializedCommand, error) { + return nil, nil +} + +// TODO: +func (s *State) SaveCommand(chatID int64, _ *SerializedCommand) error { + return nil +} + +type Bot struct { + state *State + command map[int64]Command +} + +func (b *Bot) fetchCommand(chatID int64) Command { + cmd := b.command[chatID] + if cmd == nil { + s, err := b.state.LoadCommand(chatID) + if err == nil { + cmd, err = s.AsCommand() + } + if err != nil { + cmd = nil + log.Printf("INTERNAL ERROR: couldn't fetch command for chat %d: %v", chatID, err) + } + if cmd == nil { + cmd = CommandsTemplate.DefaultCommand("") + } + } + b.command[chatID] = cmd + return cmd +} + +func (b *Bot) updateCommand(chatID int64, cmd Command) error { + b.command[chatID] = cmd + var s *SerializedCommand + if cmd != nil { + s = cmd.Serialize() + } + return b.state.SaveCommand(chatID, s) +} + +func (b *Bot) Update(u *Update) (err error) { + chatId, _ := u.ChatId() + if err != nil { + return err + } + + // Try surfacing UserError and update the bot accordingly on internal error. + defer func() { + var e UserError + if errors.As(err, &e) { + err = e.Surface(b.state) + } + if err == nil { + return + } + log.Printf("INTERNAL ERROR: %v", err) + // Reset command so that user wouldn't be stuck with internal errors + // with no way out. + if upErr := b.updateCommand(chatId, nil); upErr != nil { + log.Printf("INTERNAL ERROR: updateCommand(%d, nil): %v; while handling %v", chatId, upErr, err) + } + }() + + if u.CallbackQuery != nil { + for _, c := range CommandsTemplate.Callbacks { + if c.Match(b.state, u.CallbackQuery) { + return c.Call(b.state, u.CallbackQuery) + } + } + return fmt.Errorf("INTERNAL ERROR: Did find a corresponding callback for callback query: %v", u.CallbackQuery) + } + + if u.Message == nil { + // TODO: Make internal error type. + return fmt.Errorf("INTERNAL ERROR: Update is neither a message, nor a callback query: %v", u) + } + + // Update is a Message. + msg := u.Message.Text + for n, f := range CommandsTemplate.Commands { + if msg == n { + cmd := f(n) + cmd, err = cmd.OnCommand(b.state, u.Message) + if err != nil { + return err + } + if cmd == nil { + cmd = CommandsTemplate.DefaultCommand("") + } + return b.updateCommand(chatId, cmd) + } + } + + // None of the commands match, so process the message. + cmd := b.fetchCommand(chatId) + cmd, err = cmd.ProcessMessage(b.state, u.Message) + // On user caused error command should still be updated accordingly. + if err == nil || errors.Is(err, UserError{}) { + return b.updateCommand(chatId, cmd) + } + return err +} + +type SerializedCommand struct { + // Name of the command + Name string + // Serialized command's state. Command should be able to restore it's state + // from this string. + Data []byte +} + +func (s *SerializedCommand) AsCommand() (Command, error) { + factory := CommandsTemplate.DefaultCommand + if s == nil { + return factory(""), nil + } + for n, f := range CommandsTemplate.Commands { + if n == s.Name { + factory = f + } + } + cmd := factory(s.Name) + return cmd, cmd.Init(s) +} + +type Command interface { + Serialize() *SerializedCommand + Init(*SerializedCommand) error + // Process message from the user. + ProcessMessage(*State, *Message) (Command, error) + // Called when user runs this command. + OnCommand(*State, *Message) (Command, error) + // Both OnCommand and ProcessMessage return the new context command +} + +// Factory is used so that Command metadata is not changed accidentally. +type CommandFactory func(name string) Command + +// UserError is an error that should be surfaced to the user. +type UserError struct { + Err error + ChatID int64 +} + +func (u UserError) Error() string { + return u.Err.Error() +} + +// Surface surfaces error to the user. +func (u UserError) Surface(s *State) error { + e := u.Error() + if len(e) > 0 { + e = strings.ToUpper(e[:1]) + e[1:] + } + return s.Telegram.SendTextMessage(u.ChatID, e) +} + +type question struct { + name string + ask func(s *State, chatID int64) error + validate func(*State, *Message) error + answer string +} + +type multiQuestionCommand struct { + name string + // All questions should have unique names. + // range is used instead of map so that the called can control questions' order. + questions []*question + // Once user answers all questions save will be called. questions will have + // answers populated. + save func(state *State, chatID int64, questions []*question) error + // Name of the last question we have asked. + lastQuestion string +} + +func MultiQuestionCommandFactory(questions []*question, save func(state *State, chatID int64, questions []*question) error) CommandFactory { + return func(name string) Command { + return &multiQuestionCommand{ + name: name, + questions: questions, + save: save, + } + } +} + +type multiQuestionCommandSerialized struct { + answers map[string]string + lastQuestion string +} + +func (c *multiQuestionCommand) Serialize() *SerializedCommand { + // No need to serialize question names, they should be the same in CommandsTemplate. + // Need to serialize answers to the questions though. + a := make(map[string]string) + for _, q := range c.questions { + a[q.name] = q.answer + } + cs := &multiQuestionCommandSerialized{ + answers: a, + lastQuestion: c.lastQuestion, + } + b, err := json.Marshal(cs) + if err != nil { + log.Printf("INTERNAL ERROR: Couldn't serialize %v: %v", cs, err) + } + return &SerializedCommand{ + Name: c.name, + Data: b, + } +} + +func (c *multiQuestionCommand) Init(s *SerializedCommand) error { + cs := &multiQuestionCommandSerialized{} + if err := json.Unmarshal(s.Data, &cs); err != nil { + return fmt.Errorf("Unmarshal(%s): %w", s.Data, err) + } + for _, q := range c.questions { + q.answer = cs.answers[q.name] + } + c.lastQuestion = cs.lastQuestion + return nil +} + +func (c *multiQuestionCommand) OnCommand(s *State, m *Message) (Command, error) { + chatID := m.Chat.Id + if len(c.questions) == 0 { + // With to questions, no reason to have this command process messages. + return nil, c.save(s, chatID, c.questions) + } + if err := c.questions[0].ask(s, chatID); err != nil { + return nil, err + } + c.lastQuestion = c.questions[0].name + return c, nil +} + +func (c *multiQuestionCommand) ProcessMessage(s *State, m *Message) (Command, error) { + var q *question + for _, qe := range c.questions { + if qe.name == c.lastQuestion { + q = qe + break + } + } + if q == nil { + return nil, fmt.Errorf("INTERNAL ERROR: Did not find a question corresponding to last question %s", c.lastQuestion) + } + if err := q.validate(s, m); err != nil { + // In case validate fails with user error, we want to be able to retry, + // so we return c to be a new command. + return c, err + } + q.answer = m.Text + + var next *question = nil + for _, qe := range c.questions { + if qe.answer == "" { + next = qe + break + } + } + if next == nil { + err := c.save(s, m.Chat.Id, c.questions) + // After all questions have been answered there is no point in + // keeping trying to save, even if it fails with UserError. + return nil, err + } + if err := next.ask(s, m.Chat.Id); err != nil { + // ask should never fail with user error. + return nil, err + } + c.lastQuestion = next.name + return c, nil +} +func ReplyCommand(reply func(s *State, chatID int64) error) CommandFactory { + return MultiQuestionCommandFactory( + nil, + func(s *State, chatID int64, _ []*question) error { + return reply(s, chatID) + }, + ) +} + +func NewMessageReply(chatID int64, text string, callbacks []Callback) *MessageReply { + var ik []*InlineKeyboard + for _, c := range callbacks { + ik = append(ik, c.AsInlineKeyboard()) + } + var rm *ReplyMarkup + if len(ik) > 0 { + rm = &ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{ik}, + } + } + return &MessageReply{ + ChatId: chatID, + Text: text, + ReplyMarkup: rm, + } +} + +// practiceReply sends practice card to the user. +func practiceReply(s *State, chatID int64) error { + switch UsePractice { + case PracticeKnowledge: + default: + panic(fmt.Sprintf("INTERNAL: Unimplemented practice type: %v", UsePractice)) + } + word, err := s.Repetitions.RepeatWord(chatID) + if err == sql.ErrNoRows { + // FIXME: Make this user error instead. + return s.Telegram.SendTextMessage(chatID, "No more rows to practice; exiting practice mode.") + } + if err != nil { + return fmt.Errorf("retrieving word for repetition: %w", err) + } + return s.Telegram.SendMessage(NewMessageReply(chatID, word, []Callback{KnowCallback{word}, DontKnowCallback{word, true}})) +} + +// settingsReply sends current settings and instructions on how to change them. +func settingsReply(state *State, chatID int64) error { + s, err := state.Settings.Get(chatID) + if err != nil { + return err + } + var ls []string + for l, v := range s.TranslationLanguages { + if v { + ls = append(ls, fmt.Sprintf("%q", l)) + } + } + sort.Strings(ls) + var cmds []string + for k, _ := range SettingsCommands { + cmds = append(cmds, " "+k) + } + sort.Strings(cmds) + msg := fmt.Sprintf(` +Current settings: + +Input language: %q +Input language in ISO 639-3: %q +Translation languages in ISO 639-3: %s +Time Zone: %s + +To modify settings use one of the commands below: +%s +`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ","), s.TimeZone, strings.Join(cmds, "\n")) + return state.Telegram.SendMessage(NewMessageReply(chatID, msg, nil)) +} + +// This inteface is a bit redundant. We need it though to avoid initialization +// loop with SettingsCommands depending on settingsReply and settingsReply +// depending on SettingsCommands. +type SimpleQuestionCommand interface { + Ask(_ *State, chatID int64) error + Validate(*State, *Message) error + Save(_ *State, chatID int64, answer string) error +} + +func SimpleQuestionCommandFactory(c SimpleQuestionCommand) CommandFactory { + return MultiQuestionCommandFactory( + []*question{{ + name: "question", + ask: c.Ask, + validate: c.Validate, + }}, + func(s *State, chatID int64, questions []*question) error { + return c.Save(s, chatID, questions[0].answer) + }, + ) +} + +type SimpleSettingCommand struct { + question string + validate func(s *State, answer string) error + save func(s *State, chatID int64, answer string) error +} + +func (c *SimpleSettingCommand) Ask(s *State, chatID int64) error { + return askQuestion(c.question)(s, chatID) +} + +func (c *SimpleSettingCommand) Validate(s *State, m *Message) error { + if err := c.validate(s, m.Text); err != nil { + return UserError{ChatID: m.Chat.Id, Err: fmt.Errorf("%w. Please try again.", err)} + } + return nil +} + +func (c *SimpleSettingCommand) Save(s *State, chatID int64, answer string) error { + if err := c.save(s, chatID, answer); err != nil { + return err + } + return settingsReply(s, chatID) +} + +func askQuestion(q string) func(s *State, chatID int64) error { + return func(s *State, chatID int64) error { + return s.Telegram.SendTextMessage(chatID, q) + } +} + +func AddCommandFactory() CommandFactory { + noopValidate := func(*State, *Message) error { return nil } + return MultiQuestionCommandFactory( + []*question{{ + name: "front", + ask: askQuestion("Enter front of the card (word, expression, question)."), + validate: noopValidate, + }, { + name: "back", + ask: askQuestion("Enter back of the card (definition, answer)."), + validate: noopValidate, + }}, + func(s *State, chatID int64, qs []*question) error { + var front string + var back string + for _, q := range qs { + switch q.name { + case "front": + front = q.answer + case "back": + back = q.answer + default: + return fmt.Errorf("unexpected question in save: %v", q) + } + } + if err := s.Repetitions.Save(chatID, front, back); err != nil { + return err + } + return s.Telegram.SendTextMessage(chatID, fmt.Sprintf("Added %q for learning!", front)) + }, + ) +} + +func DeleteCommandFactory() CommandFactory { + return MultiQuestionCommandFactory( + []*question{{ + name: "word", + ask: askQuestion("Enter the word you want to delete from learning!"), + validate: func(s *State, m *Message) error { + e, err := s.Repetitions.Exists(m.Chat.Id, m.Text) + if err != nil { + return err + } + if !e { + return UserError{ChatID: m.Chat.Id, Err: fmt.Errorf("Word %q isn't saved for learning!", m.Text)} + } + return nil + }, + }}, + func(s *State, chatID int64, qs []*question) error { + if err := s.Repetitions.Delete(chatID, qs[0].answer); err != nil { + return err + } + return s.Telegram.SendTextMessage(chatID, fmt.Sprintf("Deleted %q!", qs[0].answer)) + }, + ) +} + +type defaultCommand struct{} + +func (defaultCommand) Serialize() *SerializedCommand { + return nil +} +func (defaultCommand) Init(*SerializedCommand) error { + return nil +} +func (defaultCommand) ProcessMessage(s *State, m *Message) (Command, error) { + chatID := m.Chat.Id + + if len(strings.Split(m.Text, " ")) > 1 { + return nil, UserError{ChatID: chatID, Err: fmt.Errorf("For now this bot doesn't work with expressions. Try entering a single work without spaces.")} + } + + def, err := s.Repetitions.GetDefinition(m.Chat.Id, m.Text) + if err == nil { + return nil, s.Telegram.SendMessage(NewMessageReply( + m.Chat.Id, def, + []Callback{ResetProgressCallback{m.Text}})) + } + if err != sql.ErrNoRows { + log.Printf("ERROR: Repetitions(%d, %s): %v", m.Chat.Id, m.Text, err) + } + settings, err := s.Settings.Get(chatID) + if err != nil { + return nil, fmt.Errorf("get settings: %v", err) + } + ds, err := s.Definer.Define(m.Text, settings) + if err != nil { + // TODO: Might be good to post debug logs to the reply in the debug mode. + log.Printf("Error fetching the definition: %w", err) + // TODO: Add search url to the reply? + return nil, UserError{ + ChatID: m.Chat.Id, + Err: fmt.Errorf("Couldn't find definitions."), + } + } + for _, d := range ds { + if err := s.Telegram.SendMessage(&MessageReply{ + ChatId: m.Chat.Id, + Text: d, + ParseMode: "MarkdownV2", + ReplyMarkup: &ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ + LearnCallback{m.Text}.AsInlineKeyboard(), + }}, + }, + }); err != nil { + return nil, err + } + } + return nil, nil +} + +// Should never be called. +func (defaultCommand) OnCommand(*State, *Message) (Command, error) { + return nil, nil +} + +func joinCommands(m1, m2 map[string]CommandFactory) map[string]CommandFactory { + r := make(map[string]CommandFactory) + for k, v := range m1 { + if _, dup := m2[k]; dup { + panic(fmt.Sprintf("Found duplicated command: %q", k)) + } + r[k] = v + } + for k, v := range m2 { + // There can be no additional duplications so no need to check it. + r[k] = v + } + return r +} + +// textReply sends a message and resets state. +func textReply(text string) CommandFactory { + return ReplyCommand(func(s *State, chatID int64) error { + return s.Telegram.SendTextMessage(chatID, text) + }) +} + +// SettingsCommands contains all settings-related commands. They are bundled +// together for convenience to have everything in one place. +var SettingsCommands = map[string]CommandFactory{ + "/language": SimpleQuestionCommandFactory(&SimpleSettingCommand{ + question: func() string { + var ls []string + for l, _ := range SupportedInputLanguages { + ls = append(ls, fmt.Sprintf("%q", l)) + } + sort.Strings(ls) + return fmt.Sprintf("Enter input language of your choice. Supported are %s", + strings.Join(ls, ",")) + }(), + validate: func(s *State, answer string) error { + return s.Settings.ValidateLanguage(answer) + }, + save: func(s *State, chatID int64, answer string) error { + return s.Settings.SetLanguage(chatID, answer) + }, + }), + "/timezone": SimpleQuestionCommandFactory(&SimpleSettingCommand{ + question: "Input your timezone in one of the formats: UTC, UTC+X or UTC-X.", + validate: func(s *State, answer string) error { + return s.Settings.ValidateTimeZone(answer) + }, + save: func(s *State, chatID int64, answer string) error { + return s.Settings.SetTimeZone(chatID, answer) + }, + }), +} + +var CommandsTemplate = struct { + // When receiving a callback it will be matched here. + // Tests should test that each callback is reachable. + // Callbacks can rely only on info got from the CallbackQuery, + // Everything stored in callback should be used only in AsInlineKeyboard method. + Callbacks []Callback + // Each key in the map should directly correspond to the name of the command. + // Possible improvement would be to allow regexps in the Commands name so + // arguments can be passed there. + Commands map[string]CommandFactory + // Command returned by DefaultCommand shouldn't implement OnCommand. It + // should not be ever called. + DefaultCommand CommandFactory +}{ + Commands: joinCommands( + map[string]CommandFactory{ + "/start": textReply( + "Welcome to the language bot. Still in development. No instructions " + + "so far. " + + "All sentences and translations are from Tatoeba's (https://tatoeba.org) " + + "dataset, released under a CC-BY 2.0 FR."), + "/stop": textReply("Stopped. Input the word to get it's definition."), + "/practice": ReplyCommand(practiceReply), + "/settings": ReplyCommand(settingsReply), + "/add": AddCommandFactory(), + "/delete": DeleteCommandFactory(), + }, + SettingsCommands, + ), + Callbacks: []Callback{ + KnowCallback{}, + DontKnowCallback{}, + LearnCallback{}, + }, + DefaultCommand: func(string) Command { return defaultCommand{} }, +} diff --git a/definer.go b/definer.go index 456afe3..ec56aca 100644 --- a/definer.go +++ b/definer.go @@ -66,6 +66,7 @@ func (d *Definer) Define(word string, settings *Settings) (ds []string, err erro ex, err := d.usage.FetchExamples(word, settings.InputLanguageISO639_3, settings.TranslationLanguages) if err != nil { ex = nil + log.Printf("ERROR: FetchExamples(%s): %v", word, err) log.Printf("WARNING Did not find usage examples for %q", word) } msg := "*" + escapeMarkdown(word) + "*\n" diff --git a/e2e_test.go b/e2e_test.go index c98bbd8..56ca095 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "database/sql" "encoding/json" "fmt" "io/ioutil" @@ -87,7 +88,7 @@ b:Don't know /settings -b:Input Language +/language NotARealLanguage @@ -97,15 +98,17 @@ English /settings -b:Input Language +/language Hungarian -b:Input Language +/language /stop -/delete falu +/delete + +falu /practice @@ -116,6 +119,7 @@ b:Input Language /add cardfront + cardback (definitions or what not) cardfront @@ -128,17 +132,24 @@ cardfront t.Fatal(err) } defer os.RemoveAll(dir) - db := filepath.Join(dir, "tmpdb") + dbPath := filepath.Join(dir, "tmpdb") + + // Initiate db with some usage examples. + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(usageSQL); err != nil { + t.Fatal(err) + } fk := startFakeTelegram(t) defer fk.server.Close() tm := &Telegram{hc: *fk.server.Client()} c, err := NewCommander(tm, &CommanderOptions{ - useCache: true, - dbPath: db, - sentencesPath: "./testdata/sentences.csv", - linksPath: "./testdata/links.csv", + useCache: true, + dbPath: dbPath, stages: []time.Duration{ 0, 2 * time.Minute, diff --git a/go.mod b/go.mod index ac866e5..e2a27a8 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/google/go-cmp v0.4.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 ) diff --git a/go.sum b/go.sum index ba235cf..9045755 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnA golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 71a6caf..23258ff 100644 --- a/main.go +++ b/main.go @@ -43,10 +43,8 @@ func main() { log.Printf("db_path: %q", *db) ctx := context.Background() opts := &CommanderOptions{ - useCache: false, - dbPath: *db, - sentencesPath: "./data/sentences.csv", - linksPath: "./data/links.csv", + useCache: false, + dbPath: *db, stages: []time.Duration{ 20 * time.Second, 1 * time.Hour * 23, diff --git a/migrate/Dockerfile b/migrate/Dockerfile new file mode 100644 index 0000000..ae772fa --- /dev/null +++ b/migrate/Dockerfile @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and then copy over the neede parts to create a small image +FROM golang:alpine AS builder + +RUN apk update && apk add --no-cache git gcc g++ ca-certificates apache2-utils +WORKDIR /go/src/words/migrate +COPY migrate/*.go ./ +RUN go get -d -v -tags netgo -installsuffix netgo +# netgo and ldflags makes sure that dns resolver and binary are statically +# linked giving the ability for smaller images. +RUN go build -tags netgo -installsuffix netgo -ldflags '-extldflags "-static"' -o /go/bin/migrate + +FROM scratch +# FIXME: Copying certificates looks a little bit hacky. Is there a better +# solution? +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /go/bin/migrate /go/bin/migrate +COPY data/*.csv data/ +# TODO: This should be moved to the CMD starting it up. +ENTRYPOINT ["/go/bin/migrate", "--db_path=/words-vol/db/db.sql", "--sentences=data/sentences.csv", "--links=data/links.csv"] diff --git a/migrate/README.md b/migrate/README.md new file mode 100644 index 0000000..878ab27 --- /dev/null +++ b/migrate/README.md @@ -0,0 +1,19 @@ +From the root of the project: + +```shell +sudo docker build -t loader . -f migrate/Dockerfile +sudo docker run --rm --name loader --mount source=words-vol,target=/words-vol/db/ loader +``` + +To save image: + +```shell +sudo docker save loader -o loader_image.tar +sudo chmod +r loader_image.tar +``` + + +Test image changes: +```shell +sudo docker run --rm --name temp_bash --mount source=words-vol,target=/words-vol/db/ -it ubuntu /bin/bash +``` diff --git a/migrate/load.go b/migrate/load.go new file mode 100644 index 0000000..3ed059e --- /dev/null +++ b/migrate/load.go @@ -0,0 +1,333 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// This is a script to load up usage examples into the database. +package main + +import ( + "bufio" + "database/sql" + "flag" + "fmt" + "log" + "os" + "strconv" + "strings" + "sync" + + _ "github.com/mattn/go-sqlite3" + "golang.org/x/sync/errgroup" +) + +type UsageFetcherOptions struct { + // Path to the file in csv with links between files, + LinksPath string + // Path to the file in csv with all the sentences. + // is an ISO 639-3 language code. + SentencesPath string +} + +type wordLang struct { + word string + lang string +} + +type sentence struct { + text string + lang string +} + +func (l *Loader) ReadAndLoad(opts UsageFetcherOptions) error { + sf, err := os.Open(opts.SentencesPath) + if err != nil { + return err + } + defer sf.Close() + + // Use single proc so that tx is single. Count and flush (commit + // transaction & create a new one) every 1M rows. + // proc would have sentence, word, translation methods. queries will be + // embedded. + p, err := newProc(l) + if err != nil { + return err + } + defer p.cleanup() + + c := make(chan string) + + processSentence := func() error { + for row := range c { + s := strings.Split(row, "\t") + if len(s) != 3 { + return fmt.Errorf("reading %q: wrond format for row %s", opts.SentencesPath, s) + } + id, err := strconv.ParseInt(s[0], 10, 64) + if err != nil { + return fmt.Errorf("reading %q: parsing id %q: %v", opts.SentencesPath, s[0], err) + } + lang, text := s[1], s[2] + if err := p.sentence(id, lang, text); err != nil { + return err + } + r := strings.NewReplacer( + ",", "", + ".", "", + "!", "", + ")", "", + "(", "", + "}", "", + "{", "", + "]", "", + "[", "", + ) + for _, w := range strings.Split(text, " ") { + word := strings.ToLower(r.Replace(w)) + if err := p.word(word, lang, id); err != nil { + return err + } + } + } + return nil + } + + scanner := bufio.NewScanner(sf) + go func() { + for scanner.Scan() { + c <- scanner.Text() + } + close(c) + }() + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading %q: %w", opts.SentencesPath, err) + } + eg := errgroup.Group{} + for n := 0; n < 16; n++ { + eg.Go(processSentence) + } + if err := eg.Wait(); err != nil { + return err + } + + lf, err := os.Open(opts.LinksPath) + if err != nil { + return err + } + defer lf.Close() + + scanner = bufio.NewScanner(lf) + for scanner.Scan() { + var ids []int64 + for _, i := range strings.Split(scanner.Text(), "\t") { + id, err := strconv.ParseInt(i, 10, 64) + if err != nil { + return fmt.Errorf("reading %q: parsing id %q: %v", opts.LinksPath, i, err) + } + ids = append(ids, id) + } + if len(ids) != 2 { + return fmt.Errorf("reading %q: wrond format for row %s", opts.LinksPath, scanner.Text()) + } + if err := p.translation(ids[0], ids[1]); err != nil { + return err + } + } + + return nil +} + +type Loader struct { + db *sql.DB + opts UsageFetcherOptions +} + +func NewLoader(dbPath, sPath, lPath string) (*Loader, error) { + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, fmt.Errorf("open: %v", err) + } + return &Loader{ + db, + UsageFetcherOptions{ + SentencesPath: sPath, + LinksPath: lPath, + }, + }, nil +} + +type proc struct { + db *sql.DB + stmt map[TableType]*sql.Stmt + + mu sync.Mutex + cnt map[TableType]int + processed int64 + tx *sql.Tx +} + +type TableType int + +const ( + SentencesTable TableType = iota + WordsTable + TranslationsTable +) + +func newProc(l *Loader) (p *proc, err error) { + p = new(proc) + p.tx, err = l.db.Begin() + if err != nil { + return + } + p.db = l.db + p.cnt = make(map[TableType]int) + p.stmt = make(map[TableType]*sql.Stmt) + for t, q := range map[TableType]string{ + SentencesTable: `INSERT OR REPLACE INTO Sentences(id, lang, text) + VALUES(?, ?, ?)`, + WordsTable: `INSERT OR REPLACE INTO Words(word, lang, sentence_id) + VALUES(?, ?, ?)`, + TranslationsTable: `INSERT OR REPLACE INTO Translations(id, translation_id) + VALUES(?, ?)`, + } { + p.stmt[t], err = l.db.Prepare(q) + if err != nil { + return + } + } + return +} + +func (p *proc) sentence(id int64, lang, text string) error { + err := p.row(SentencesTable, id, lang, text) + if err != nil { + err = fmt.Errorf("Row(%d, %s, %s): %v", id, lang, text, err) + } + return err +} +func (p *proc) word(word, lang string, sid int64) error { + err := p.row(WordsTable, word, lang, sid) + if err != nil { + err = fmt.Errorf("Row(%s, %s, %d): %v", word, lang, sid, err) + } + return err +} +func (p *proc) translation(id, tid int64) error { + err := p.row(TranslationsTable, id, tid) + if err != nil { + err = fmt.Errorf("Row(%d, %d): %v", id, tid, err) + } + return err +} + +func (p *proc) row(table TableType, args ...interface{}) error { + p.mu.Lock() + defer p.mu.Unlock() + p.cnt[table] += 1 + _, err := p.tx.Stmt(p.stmt[table]).Exec(args...) + if err != nil { + return err + } + p.processed += 1 + if p.processed%100_000 == 0 { + return p.commit() + } + return nil +} + +func (p *proc) commit() (err error) { + log.Printf("Flushing %d rows", p.processed) + if err := p.tx.Commit(); err != nil { + return err + } + log.Printf("In total wrote %v", p.cnt) + p.processed = 0 + p.tx, err = p.db.Begin() + return err +} + +func (p *proc) cleanup() { + if err := p.commit(); err != nil { + log.Printf("ERROR proc cleanup: %v", err) + } + for _, s := range p.stmt { + s.Close() + } + p.tx.Rollback() +} + +func (l *Loader) Load() error { + // word -> list of sentences (ids). OR word -> lang -> list of sentences. + // sentence id -> list of translation id. + // translation id -> sentence. + if _, err := l.db.Exec(` + PRAGMA foreign_keys = OFF; + + CREATE TABLE IF NOT EXISTS Sentences ( + id INTEGER PRIMARY KEY, + lang STRING, + text STRING + ); + + CREATE TABLE IF NOT EXISTS Translations ( + id INTEGER, + translation_id INTEGER, + FOREIGN KEY(id) REFERENCES Sentences(id), + FOREIGN KEY(translation_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS TranslationsIdIndex + ON Translations (id); + + CREATE TABLE IF NOT EXISTS Words ( + word STRING, + lang STRING, + sentence_id INTEGER, + FOREIGN KEY(sentence_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS WordLangIndex + ON Words (word, lang); + `); err != nil { + return err + } + + if err := l.ReadAndLoad(l.opts); err != nil { + return err + } + + //{ + // p, err := newProc(l, + // `INSERT OR REPLACE INTO Sentences(id, lang, text) + // VALUES(?, ?, ?)`) + // if err != nil { + // return err + // } + // defer p.cleanup() + return nil +} + +func main() { + db := flag.String("db_path", "../db.sql", "Path to the persistent sqlite3 database.") + sentences := flag.String("sentences", "../data/sentences.csv", "Path to the folder with sentences usage examples in csv format.") + links := flag.String("links", "../data/links.csv", "Path to the folder with links usage examples in csv format.") + flag.Parse() + + l, err := NewLoader(*db, *sentences, *links) + if err != nil { + log.Fatal(err) + } + if err := l.Load(); err != nil { + log.Fatal(err) + } +} diff --git a/migrate/load_test.go b/migrate/load_test.go new file mode 100644 index 0000000..31f67fe --- /dev/null +++ b/migrate/load_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLoad(t *testing.T) { + dir, err := ioutil.TempDir("", "load_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + + count := func(table string) int32 { + t.Helper() + r := db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s;", table)) + var n int32 + if err := r.Scan(&n); err != nil { + t.Fatalf("count(%s): %v", table, err) + } + return n + } + + l, err := NewLoader(dbPath, "../testdata/sentences.csv", "../testdata/links.csv") + if err != nil { + t.Fatal(err) + } + if err := l.Load(); err != nil { + t.Fatal(err) + } + tables := []string{"Sentences", "Translations", "Words"} + got := make(map[string]int32) + for _, tb := range tables { + n := count(tb) + if n == 0 { + t.Errorf("%s: got %d want > 0", tb, n) + } + got[tb] = n + } + + // Concecutive calls to Load should result in no errors and be noops. + if err := l.Load(); err != nil { + t.Fatal(err) + } + want := got + for _, tb := range tables { + got[tb] = count(tb) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v want %v", got, want) + } + log.Printf("want: %v", want) +} diff --git a/migrate/migrate.go b/migrate/migrate.go deleted file mode 100644 index b92c428..0000000 --- a/migrate/migrate.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// This is a adhoc one-time run script to perform data migration on the -// database. -package main - -import ( - "database/sql" - "fmt" - "strings" - - _ "github.com/mattn/go-sqlite3" -) - -func Migrate(dbPath string) error { - db, err := sql.Open("sqlite3", dbPath) - if err != nil { - return fmt.Errorf("open: %v", err) - } - - rows, err := db.Query(` - SELECT word, definition - FROM Repetition;`) - defer rows.Close() - - nd := make(map[string]string) - for rows.Next() { - var w, d string - if err := rows.Scan(&w, &d); err != nil { - return fmt.Errorf("scan: %v", err) - } - nd[w] = strings.ReplaceAll(d, "********", w) - if !strings.HasPrefix(d, w) && !strings.HasPrefix(d, "*"+w) { - nd[w] = "*" + w + "*\n\n" + nd[w] - } - } - rows.Close() - - for w, d := range nd { - _, err := db.Exec(` - UPDATE Repetition - SET definition = $0 - WHERE word = $1;`, - d, w) - if err != nil { - return fmt.Errorf("update: %v", err) - } - } - return nil -} - -func main() { - fmt.Printf("Result of migration: %v\n", Migrate("../db.sql")) -} diff --git a/reminder.go b/reminder.go new file mode 100644 index 0000000..b2750e1 --- /dev/null +++ b/reminder.go @@ -0,0 +1,114 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "fmt" + "log" + "time" +) + +type Notification struct { + ChatID int64 +} + +// reminder +type Reminder struct { + sendNofication func(*Notification) error + fetchSettings func() (map[int64]*Settings, error) + + // db stores last reminder time for each chat ID. + db *sql.DB +} + +func NewReminder(c *Clients, db *sql.DB) (*Reminder, error) { + + if _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS Reminders ( + chat_id INTEGER PRIMARY KEY, + last_reminder_time_seconds INTEGER -- seconds since UNIX epoch + );`); err != nil { + return nil, err + } + + return &Reminder{ + db: db, + sendNofication: func(n *Notification) error { + log.Printf("Notification: %v", n) + return nil + }, + fetchSettings: c.Settings.GetAll, + }, nil +} + +func (r *Reminder) LastReminderTime(chatID int64) (time.Time, error) { + row := r.db.QueryRow(` + SELECT last_reminder_time_seconds + FROM Reminders + WHERE chat_id = $0`, + chatID) + var u int64 + err := row.Scan(&u) + if err != nil { + u = 0 + if err != sql.ErrNoRows { + err = fmt.Errorf("INTERNAL: retrieving last_reminder_time_seconds for chat id %d: %w", chatID, err) + } + } + return time.Unix(u, 0), err +} + +func (r *Reminder) UpdateLastReminderTime(chatID int64) error { + _, err := r.db.Exec(` + INSERT OR REPLACE INTO Reminders(chat_id, last_reminder_time_seconds) VALUES + ($0, $1);`, + chatID, time.Now().Unix()) + if err != nil { + return fmt.Errorf("INTERNAL: Failed updating reminder_time: %w", err) + } + return nil +} + +func (r *Reminder) Loop(ticker <-chan time.Time, cancel <-chan struct{}) { + for { + cs, err := r.fetchSettings() + if err != nil { + log.Printf("ERROR: fetchSettings: %v", err) + } + // TODO: Take into account availability window, reminder frequency and timezone. + // newReminderTime = lastReminder + (aval window size)/Frequency + for chatID, _ := range cs { + const frequency = 1 + rt, err := r.LastReminderTime(chatID) + if err != nil { + log.Print(err) + } + newRT := rt.Add(24 / frequency * time.Hour) + if time.Now().After(newRT) { + if err := r.sendNofication(&Notification{chatID}); err != nil { + log.Print(err) + } + if err := r.UpdateLastReminderTime(chatID); err != nil { + log.Print(err) + } + } + } + select { + case <-ticker: + case <-cancel: + return + } + } +} diff --git a/reminder_test.go b/reminder_test.go new file mode 100644 index 0000000..f639a85 --- /dev/null +++ b/reminder_test.go @@ -0,0 +1,78 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +func TestReminders(t *testing.T) { + dir, err := ioutil.TempDir("", "repetition") + if err != nil { + t.Fatal(err) + } + t.Logf("Temp dir: %q", dir) + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + + settings, err := NewSettingsConfig(dbPath) + if err != nil { + t.Fatal(err) + } + + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + + r, err := NewReminder(&Clients{ + Settings: settings, + }, db) + if err != nil { + t.Fatal(err) + } + + const chatID int64 = 0 + if err := settings.Set(chatID, DefaultSettings()); err != nil { + t.Fatal(err) + } + + c := make(chan time.Time) + + cancel := make(chan struct{}) + go func() { + c <- time.Now() + cancel <- struct{}{} + }() + + var sent []*Notification + r.sendNofication = func(n *Notification) error { + sent = append(sent, n) + return nil + } + + r.Loop(c, cancel) + + if len(sent) != 1 { + t.Errorf("got %d notifications (%v), want 1", len(sent), sent) + } +} diff --git a/settings.go b/settings.go index 851de23..a099a01 100644 --- a/settings.go +++ b/settings.go @@ -30,6 +30,9 @@ type Settings struct { // which to accept the translations. // true if translation is accepted TranslationLanguages map[string]bool + // TimeZone should be from IANA time zone database, e. g. America/New_York + // https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + TimeZone string } func SettingsFromString(s string) *Settings { @@ -49,6 +52,7 @@ func DefaultSettings() *Settings { "rus": true, "ukr": true, }, + TimeZone: "UTC", } } @@ -79,6 +83,33 @@ func NewSettingsConfig(dbPath string) (*SettingsConfig, error) { return &SettingsConfig{db}, nil } +func (c *SettingsConfig) GetAll() (map[int64]*Settings, error) { + rows, err := c.db.Query(` + SELECT chat_id, settings + FROM Settings`) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + defer rows.Close() + + r := make(map[int64]*Settings) + + for rows.Next() { + var ( + chatID int64 + s string + ) + if err := rows.Scan(&chatID, &s); err != nil { + return nil, err + } + r[chatID] = SettingsFromString(s) + } + return r, nil +} + func (c *SettingsConfig) Get(chatID int64) (*Settings, error) { row := c.db.QueryRow(` SELECT settings @@ -105,3 +136,46 @@ func (c *SettingsConfig) Set(chatID int64, s *Settings) error { } return nil } + +func (c *SettingsConfig) ValidateLanguage(language string) error { + _, ok := SupportedInputLanguages[language] + if !ok { + return fmt.Errorf("unsupported language %q", language) + } + return nil +} + +func (c *SettingsConfig) SetLanguage(chatid int64, language string) error { + currentSettings, err := c.Get(chatid) + if err != nil { + return err + } + if err := c.ValidateLanguage(language); err != nil { + return err + } + languageSettings := SupportedInputLanguages[language] + currentSettings.InputLanguage = languageSettings.InputLanguage + currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3 + currentSettings.TranslationLanguages = languageSettings.TranslationLanguages + return c.Set(chatid, currentSettings) +} + +func (c *SettingsConfig) ValidateTimeZone(tz string) error { + set := TimeZones[tz] + if !set { + return fmt.Errorf("unsupported time zone (format should be UTC, UTC+X or UTC-X)") + } + return nil +} + +func (c *SettingsConfig) SetTimeZone(chatid int64, tz string) error { + if err := c.ValidateTimeZone(tz); err != nil { + return err + } + currentSettings, err := c.Get(chatid) + if err != nil { + return err + } + currentSettings.TimeZone = tz + return c.Set(chatid, currentSettings) +} diff --git a/settings_test.go b/settings_test.go index be3a5d0..f020aee 100644 --- a/settings_test.go +++ b/settings_test.go @@ -55,4 +55,15 @@ func TestSettings(t *testing.T) { if !reflect.DeepEqual(s, ns) { t.Errorf("new settings were not set! old: %v\n new: %v", s, ns) } + + gotAll, err := settings.GetAll() + if err != nil { + t.Error(err) + } + wantAll := map[int64]*Settings{ + chatID: ns, + } + if !reflect.DeepEqual(gotAll, wantAll) { + t.Errorf("settings.GetAll() got: %v want: %v", gotAll, wantAll) + } } diff --git a/testdata/e2e.json b/testdata/e2e.json index 5b71ee1..2f5d368 100644 --- a/testdata/e2e.json +++ b/testdata/e2e.json @@ -6,12 +6,12 @@ }, { "Send": "many words", - "Want": "Couldn't process your message \"many words\"", + "Want": "For now this bot doesn't work with expressions. Try entering a single work without spaces.", "WantButtons": null }, { "Send": "oijasdki#noresults#", - "Want": "Couldn't find definitions", + "Want": "Couldn't find definitions.", "WantButtons": null }, { @@ -21,14 +21,14 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Learn" ] }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Learn" ] @@ -40,7 +40,7 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Reset progress" ] @@ -63,10 +63,8 @@ }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { "Send": "/practice", @@ -91,7 +89,7 @@ }, { "Send": "falu", - "Want": "*falu*\n\n1\\. \\[*noun*\\] village\n\nUsage examples:\n\n1\\. Minden falu egy személyt delegálhatott\\.\n\n2\\. Mindegyik falu szép volt\\.\n\n3\\. A gátépítés előtt egy falu volt itt\\.", + "Want": "*falu*\n\n1\\. \\[*noun*\\] village\nA világ egy falu\\.The world is a village\\.\nSynonym: község\nHypernyms: település, helység\nHyponyms: törpefalu \\(\u003c100\\), aprófalu \\(100–500\\), kisfalu \\(500–1,000\\), középfalu \\(1,000–2,000\\), nagyfalu \\(2,000–5,000\\), óriásfalu \\(5,000–10,000 of population\\)\nCoordinate term: város \\(town or city\\)\n\nDidn't find usage examples\\.", "WantButtons": [ "Learn" ] @@ -119,69 +117,66 @@ }, { "Send": "/stop", - "Want": "Stopped practice", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "NotARealLanguage", - "Want": "Unsupported language. Try again.", + "Want": "Unsupported language \"NotARealLanguage\". Please try again.", "WantButtons": null }, { "Send": "English", - "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { "Send": "/stop", - "Want": "Exited settings", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "Hungarian", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "/stop", - "Want": "Exited settings", + "Want": "Stopped. Input the word to get it's definition.", + "WantButtons": null + }, + { + "Send": "/delete", + "Want": "Enter the word you want to delete from learning!", "WantButtons": null }, { - "Send": "/delete falu", + "Send": "falu", "Want": "Deleted \"falu\"!", "WantButtons": null }, @@ -192,21 +187,26 @@ }, { "Send": "/add", - "Want": "\nEnter the card you want to add in the format:\n\u003cfront of the card (word, expression, question)\u003e\n\u003cback of the card (can be multiline)\u003e\n", + "Want": "Enter front of the card (word, expression, question).", "WantButtons": null }, { "Send": "/stop", - "Want": "Cancelled addition", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/add", - "Want": "\nEnter the card you want to add in the format:\n\u003cfront of the card (word, expression, question)\u003e\n\u003cback of the card (can be multiline)\u003e\n", + "Want": "Enter front of the card (word, expression, question).", + "WantButtons": null + }, + { + "Send": "cardfront", + "Want": "Enter back of the card (definition, answer).", "WantButtons": null }, { - "Send": "cardfront\ncardback (definitions or what not)", + "Send": "cardback (definitions or what not)", "Want": "Added \"cardfront\" for learning!", "WantButtons": null }, @@ -225,4 +225,4 @@ "Don't know" ] } -] +] \ No newline at end of file diff --git a/usage.go b/usage.go index 6912b27..09c86db 100644 --- a/usage.go +++ b/usage.go @@ -14,117 +14,31 @@ package main import ( - "bufio" + "database/sql" "fmt" - "log" - "os" - "sort" - "strconv" "strings" ) -type UsageFetcherOptions struct { - // Path to the file in csv with links between files, - LinksPath string - // Path to the file in csv with all the sentences. - // is an ISO 639-3 language code. - SentencesPath string -} - // Usage is struct that is able to extract usage examples from the tatoeba // datasets. type UsageFetcher struct { - // word -> sentence IDs - ws map[string][]int64 - // id -> sentence - ss map[int64]*sentence - // id -> translation ids - ts map[int64][]int64 + db *sql.DB } type sentence struct { - text string - lang string - translationIDs []int64 + text string + lang string } -func NewUsageFetcher(opts UsageFetcherOptions) (*UsageFetcher, error) { - // FIXME: Check that Language is not in TranslationLanguages - sf, err := os.Open(opts.SentencesPath) +// NewUsageFetcher creates a new usage fetcher. +func NewUsageFetcher(dbPath string) (*UsageFetcher, error) { + db, err := sql.Open("sqlite3", dbPath) if err != nil { return nil, err } - defer sf.Close() - - whitelist := make(map[string]bool) - for _, v := range SupportedInputLanguages { - whitelist[v.InputLanguageISO639_3] = true - for k, v := range v.TranslationLanguages { - if v { - whitelist[k] = true - } - } - } - - ss := make(map[int64]*sentence) - ws := make(map[string][]int64) - scanner := bufio.NewScanner(sf) - for scanner.Scan() { - s := strings.Split(scanner.Text(), "\t") - if len(s) != 3 { - return nil, fmt.Errorf("reading %q: wrond format for row %s", opts.SentencesPath, s) - } - id, err := strconv.ParseInt(s[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("reading %q: parsing id %q: %v", opts.SentencesPath, s[0], err) - } - // Skip unsupported languages and translations for optimization. Might - // need to be removed if this thing is made more general. - if !whitelist[s[1]] { - continue - } - ss[id] = &sentence{ - lang: s[1], - text: s[2], - } - // FIXME: Words will not be correctly extracted if there are - // punctuations. - for _, w := range strings.Split(s[2], " ") { - ws[w] = append(ws[w], id) - } - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading %q: %w", opts.SentencesPath, err) - } - - lf, err := os.Open(opts.LinksPath) - if err != nil { - return nil, err - } - defer lf.Close() - - ts := make(map[int64][]int64) - scanner = bufio.NewScanner(lf) - for scanner.Scan() { - var ids []int64 - for _, i := range strings.Split(scanner.Text(), "\t") { - id, err := strconv.ParseInt(i, 10, 64) - if err != nil { - return nil, fmt.Errorf("reading %q: parsing id %q: %v", opts.LinksPath, i, err) - } - ids = append(ids, id) - } - if len(ids) != 2 { - return nil, fmt.Errorf("reading %q: wrond format for row %s", opts.LinksPath, scanner.Text()) - } - ts[ids[0]] = append(ts[ids[0]], ids[1]) - } - log.Printf("Number of translations: %d", len(ts)) - + // Schema for the db can be found in migrate/load.go return &UsageFetcher{ - ws: ws, - ss: ss, - ts: ts, + db: db, }, nil } @@ -136,45 +50,68 @@ type UsageExample struct { // FIXME: Too many parameters // language is a langugage of the word in ISO 639-3 format. func (u *UsageFetcher) FetchExamples(word, language string, translationLanguages map[string]bool) ([]*UsageExample, error) { - // TODO: rank examples by complexity and extract the simplest ones: - // 1) for each word calculate it's complexity by the number of sentences it's - // used in (more sentences -> simpler words) - // 2) the sentence is simpler if it contains simpler words. Maybe average - // word simplicity to not disqualify long sentences. - // - // TODO: Prioritize using sentences with the most translations. + var tls []interface{} + for k, v := range translationLanguages { + if v { + tls = append(tls, k) + } + } + // We use Sprintf only to insert variable number of ?, so it cannot cause + // SQL injection. + q := fmt.Sprintf(` + SELECT DISTINCT s.text, ts.text + FROM + Words + INNER JOIN + Sentences s ON Words.sentence_id = s.id + LEFT JOIN + Translations ON Words.sentence_id = Translations.id + LEFT JOIN + Sentences ts ON Translations.translation_id = ts.id + WHERE + Words.word = ? + AND s.lang = ? + AND (ts.lang IS NULL OR ts.lang IN (?%s)) + -- If possible get definitions with translations first. + ORDER BY CASE WHEN ts.text IS NULL THEN 1 ELSE 0 END + LIMIT 3;`, strings.Repeat(", ?", len(tls)-1)) + args := append([]interface{}{ + word, language, + }, tls...) + rows, err := u.db.Query(q, args...) + if err == sql.ErrNoRows { + return nil, fmt.Errorf("no sentences match %s", word) + } + if err != nil { + return nil, err + } + defer rows.Close() + var ex []*UsageExample - for _, i := range u.ws[word] { - s := u.ss[i] - if s.lang != language { - continue + for rows.Next() { + var ( + e string + t sql.NullString + ) + if err := rows.Scan(&e, &t); err != nil { + return nil, err } var tr []string - for _, i := range u.ts[i] { - s, ok := u.ss[i] - if !ok { - log.Printf("ERROR: inconsistent sentences, no id %d found in sentences", i) - continue - } - if _, ok := translationLanguages[s.lang]; !ok { - continue - } - tr = append(tr, s.text) - } - if len(tr) > 4 { - tr = tr[:4] + if t.Valid { + tr = append(tr, t.String) } ex = append(ex, &UsageExample{ - Text: s.text, + Text: e, Translations: tr, }) } - if len(ex) == 0 { - return nil, fmt.Errorf("no sentences match %s", word) - } - sort.Slice(ex, func(i, j int) bool { return len(ex[i].Translations) > len(ex[j].Translations) }) - if len(ex) > 3 { - ex = ex[:3] - } + + // TODO: rank examples by complexity and extract the simplest ones: + // 1) for each word calculate it's complexity by the number of sentences it's + // used in (more sentences -> simpler words) + // 2) the sentence is simpler if it contains simpler words. Maybe average + // word simplicity to not disqualify long sentences. + // + // TODO: Prioritize using sentences with the most translations. return ex, nil } diff --git a/usage_test.go b/usage_test.go index dfc1588..497d528 100644 --- a/usage_test.go +++ b/usage_test.go @@ -14,20 +14,85 @@ package main import ( + "io/ioutil" + "os" + "path/filepath" "strings" "testing" ) +const usageSQL = ` + PRAGMA foreign_keys = OFF; + + CREATE TABLE IF NOT EXISTS Sentences ( + id INTEGER PRIMARY KEY, + lang STRING, + text STRING + ); + + CREATE TABLE IF NOT EXISTS Translations ( + id INTEGER, + translation_id INTEGER, + FOREIGN KEY(id) REFERENCES Sentences(id), + FOREIGN KEY(translation_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS TranslationsIdIndex + ON Translations (id); + + CREATE TABLE IF NOT EXISTS Words ( + word STRING, + lang STRING, + sentence_id INTEGER, + FOREIGN KEY(sentence_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS WordLangIndex + ON Words (word, lang); + + INSERT OR REPLACE INTO Sentences(id, lang, text) VALUES + (1, "hun", "fekete kutya"), + (2, "hun", "fekete disznó"), + (3, "hun", "fekete macska fehér asztalon"), + (4, "hun", "fehér disznó"), + (5, "hun", "fehér fal"), + (6, "hun", "fehér haj"), + (7, "eng", "black dog"), + (8, "eng", "white pig"), + (9, "ukr", "чорний собака"); + INSERT OR REPLACE INTO Words(word, lang, sentence_id) VALUES + ("fekete", "hun", 1), + ("fekete", "hun", 2), + ("fekete", "hun", 3), + ("fehér", "hun", 3), + ("fehér", "hun", 4), + ("fehér", "hun", 5), + ("fehér", "hun", 6); + INSERT OR REPLACE INTO Translations(id, translation_id) VALUES + (1, 7), + (1, 9), + (7, 1), + (9, 1), + (4, 8), + (8, 4); + ` + func TestUsageFetcher(t *testing.T) { - uf, err := NewUsageFetcher(UsageFetcherOptions{ - SentencesPath: "testdata/sentences.csv", - LinksPath: "testdata/links.csv", - }) + dir, err := ioutil.TempDir("", "repetition") + if err != nil { + t.Fatal(err) + } + t.Logf("Temp dir: %q", dir) + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + uf, err := NewUsageFetcher(dbPath) if err != nil { t.Fatal(err) } + if _, err := uf.db.Exec(usageSQL); err != nil { + t.Fatal(err) + } - for _, word := range []string{"fekete", "fehér", "közös"} { + for _, word := range []string{"fekete", "fehér"} { t.Run(word, func(t *testing.T) { ex, err := uf.FetchExamples(word, "hun", map[string]bool{ "eng": true, @@ -38,8 +103,8 @@ func TestUsageFetcher(t *testing.T) { t.Fatal(err) } // check that each returned example contains asked query. - if len(ex) == 0 { - t.Fatalf("len(usage examples): got %d; want > 0", len(ex)) + if len(ex) != 3 { + t.Fatalf("len(usage examples): got %d; want 3", len(ex)) } for _, e := range ex { if !strings.Contains(e.Text, word) {