From 9f41b7ec164ce3dbce77f310fe3cbeea6aeeb57a Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 8 Aug 2020 12:57:46 +0300 Subject: [PATCH 1/8] ignore vscode --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index fa76a7c..bd589bd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ commands.md cover.out testdata/e2e_corrected.json backups/* + +.vscode/ From 35ab166712c642b8fc099fd51f2d069fe0c1f4b2 Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 26 Sep 2020 21:56:13 +0300 Subject: [PATCH 2/8] add timezone setting --- Dockerfile | 2 +- README.md | 20 ++++++++++++++++ actions.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++----- commands.go | 9 ++++++++ settings.go | 28 +++++++++++++++++++++++ 5 files changed, 118 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef5398a..d368c2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ FROM golang:alpine AS builder RUN apk update && apk add --no-cache git gcc g++ ca-certificates apache2-utils WORKDIR /go/src/words COPY *.go ./ -Run go get -d -v -tags netgo -installsuffix netgo +RUN go get -d -v -tags netgo -installsuffix netgo # netgo and ldflags makes sure that dns resolver and binary are statically # linked giving the ability for smaller images. RUN go build -tags netgo -installsuffix netgo -ldflags '-extldflags "-static"' -o /go/bin/words diff --git a/README.md b/README.md index b6df726..ef70e25 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,23 @@ repository. It should live in data/links.csv and data/sentences.csv It can be downloaded from here: https://tatoeba.org/eng/downloads + +## QuickStart: +1. Create a telegram bot using @BotFather if you don't have one yet +2. Create secret.go in the root folder with the following content +```go + +package main + +const BotToken = "TOKEN PROVIDED BY BOT FATHER" +``` +3. Download links and senteces from https://tatoeba.org/eng/downloads +4. Put fetched csv data under `./data` +5. Run using one of the following: +- using Go: `go build && ./words` +- using Docker: +```bash +sudo docker volume create words-vol +sudo docker build -t words . +sudo docker run --rm --name words-app --mount source=words-vol,target=/words-vol/db/ words +``` diff --git a/actions.go b/actions.go index abcf97b..7b6df0d 100644 --- a/actions.go +++ b/actions.go @@ -207,14 +207,16 @@ Current settings: Input language: %q Input language in ISO 639-3: %q Translation languages in ISO 639-3: %s +Time Zone: %s Choose setting which you want to modify. (The choice might improve in the future.) -`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ",")) +`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ","), s.TimeZone) return []Action{ // Note that stop should be handled before input language is! &StopAction{a.Clients, "Exited settings"}, &InputLanguageButton{a.Clients}, + &TimeZoneButton{a.Clients}, &PracticeAction{a.Clients}, &CatchAllMessagesAction{CatchAllAction{ a.Clients, "type /stop to exit settings"}}, @@ -222,14 +224,21 @@ Choose setting which you want to modify. ChatId: chatId, Text: msg, ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - &InlineKeyboard{ + InlineKeyboard: [][]*InlineKeyboard{{ + { Text: "Input Language", CallbackData: CallbackInfo{ Action: ChangeSettingAction, Setting: "InputLanguage", }.String(), }, + { + Text: "Time Zone", + CallbackData: CallbackInfo{ + Action: ChangeSettingAction, + Setting: "TimeZone", + }.String(), + }, }}, }, }) @@ -266,6 +275,32 @@ func (b *InputLanguageButton) Match(u *Update) bool { return info.Setting == "InputLanguage" } +type TimeZoneButton struct { + *Clients +} + +func (b *TimeZoneButton) Perform(u *Update) ([]Action, error) { + chatId, err := u.ChatId() + if err != nil { + return nil, err + } + return []Action{ + &StopAction{b.Clients, "Exited settings"}, + &PickTimeZoneAction{b.Clients}, + }, b.Telegram.SendMessage(&MessageReply{ + ChatId: chatId, + Text: "Input your timezone in format (UTC, UTC+X or UTC-X)", + }) +} + +func (b *TimeZoneButton) Match(u *Update) bool { + if u.CallbackQuery == nil { + return false + } + info := CallbackInfoFromString(u.CallbackQuery.Data) + return info.Setting == "TimeZone" +} + type PickLanguageAction struct { *Clients } @@ -273,11 +308,10 @@ type PickLanguageAction struct { func (a *PickLanguageAction) Perform(u *Update) ([]Action, error) { m := u.Message chatId := m.Chat.Id - set, ok := SupportedInputLanguages[m.Text] - if !ok { + err := a.Settings.SetLanguage(chatId, m.Text) + if err != nil { return nil, a.Telegram.SendTextMessage(chatId, "Unsupported language. Try again.") } - a.Settings.Set(chatId, &set) sa := SettingsAction{a.Clients} return sa.Perform(u) } @@ -286,6 +320,26 @@ func (a *PickLanguageAction) Match(u *Update) bool { return u.Message != nil } +type PickTimeZoneAction struct { + *Clients +} + +func (a *PickTimeZoneAction) Perform(u *Update) ([]Action, error) { + m := u.Message + chatId := m.Chat.Id + set := TimeZones[m.Text] + if !set { + return nil, a.Telegram.SendTextMessage(chatId, "Unsupported time zone. Try again (format should be UTC, UTC+X or UTC-X).") + } + a.Settings.SetTimeZone(chatId, m.Text) + sa := SettingsAction{a.Clients} + return sa.Perform(u) +} + +func (a *PickTimeZoneAction) Match(u *Update) bool { + return u.Message != nil +} + type PracticeAction struct { *Clients } diff --git a/commands.go b/commands.go index e1de0f7..9a9a321 100644 --- a/commands.go +++ b/commands.go @@ -43,6 +43,7 @@ const UsePractice = PracticeKnowledge var ( SupportedInputLanguages map[string]Settings + TimeZones map[string]bool ) func init() { @@ -74,6 +75,14 @@ func init() { }, }, } + TimeZones = func() map[string]bool { + timeZones := make(map[string]bool) + for i := -12; i < 12; i++ { + timeZones[fmt.Sprintf("UTC%+d", i)] = true + } + timeZones["UTC"] = true + return timeZones + }() } type CallbackAction int diff --git a/settings.go b/settings.go index 851de23..aa0d91a 100644 --- a/settings.go +++ b/settings.go @@ -30,6 +30,9 @@ type Settings struct { // which to accept the translations. // true if translation is accepted TranslationLanguages map[string]bool + // TimeZone should be from IANA time zone database, e. g. America/New_York + // https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + TimeZone string } func SettingsFromString(s string) *Settings { @@ -49,6 +52,7 @@ func DefaultSettings() *Settings { "rus": true, "ukr": true, }, + TimeZone: "UTC", } } @@ -105,3 +109,27 @@ func (c *SettingsConfig) Set(chatID int64, s *Settings) error { } return nil } + +func (c *SettingsConfig) SetLanguage(chatid int64, language string) error { + currentSettings, err := c.Get(chatid) + if err == nil { + languageSettings, ok := SupportedInputLanguages[language]; + if (!ok) { + return fmt.Errorf("unsupported language %q", language) + } + currentSettings.InputLanguage = languageSettings.InputLanguage; + currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3; + currentSettings.TranslationLanguages = languageSettings.TranslationLanguages; + return c.Set(chatid, currentSettings) + } + return nil +} + +func (c *SettingsConfig) SetTimeZone(chatid int64, tz string) error { + currentSettings, err := c.Get(chatid) + if err == nil { + currentSettings.TimeZone = tz + return c.Set(chatid, currentSettings) + } + return nil +} From efce06d1f716aa02e8e00ffc2088c2404b0154b9 Mon Sep 17 00:00:00 2001 From: Attila Date: Sun, 27 Sep 2020 15:07:55 +0100 Subject: [PATCH 3/8] Create simple functionality to send practice reminders. --- reminder.go | 114 +++++++++++++++++++++++++++++++++++++++++++++++ reminder_test.go | 78 ++++++++++++++++++++++++++++++++ settings.go | 37 ++++++++++++--- settings_test.go | 11 +++++ 4 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 reminder.go create mode 100644 reminder_test.go diff --git a/reminder.go b/reminder.go new file mode 100644 index 0000000..b2750e1 --- /dev/null +++ b/reminder.go @@ -0,0 +1,114 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "fmt" + "log" + "time" +) + +type Notification struct { + ChatID int64 +} + +// reminder +type Reminder struct { + sendNofication func(*Notification) error + fetchSettings func() (map[int64]*Settings, error) + + // db stores last reminder time for each chat ID. + db *sql.DB +} + +func NewReminder(c *Clients, db *sql.DB) (*Reminder, error) { + + if _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS Reminders ( + chat_id INTEGER PRIMARY KEY, + last_reminder_time_seconds INTEGER -- seconds since UNIX epoch + );`); err != nil { + return nil, err + } + + return &Reminder{ + db: db, + sendNofication: func(n *Notification) error { + log.Printf("Notification: %v", n) + return nil + }, + fetchSettings: c.Settings.GetAll, + }, nil +} + +func (r *Reminder) LastReminderTime(chatID int64) (time.Time, error) { + row := r.db.QueryRow(` + SELECT last_reminder_time_seconds + FROM Reminders + WHERE chat_id = $0`, + chatID) + var u int64 + err := row.Scan(&u) + if err != nil { + u = 0 + if err != sql.ErrNoRows { + err = fmt.Errorf("INTERNAL: retrieving last_reminder_time_seconds for chat id %d: %w", chatID, err) + } + } + return time.Unix(u, 0), err +} + +func (r *Reminder) UpdateLastReminderTime(chatID int64) error { + _, err := r.db.Exec(` + INSERT OR REPLACE INTO Reminders(chat_id, last_reminder_time_seconds) VALUES + ($0, $1);`, + chatID, time.Now().Unix()) + if err != nil { + return fmt.Errorf("INTERNAL: Failed updating reminder_time: %w", err) + } + return nil +} + +func (r *Reminder) Loop(ticker <-chan time.Time, cancel <-chan struct{}) { + for { + cs, err := r.fetchSettings() + if err != nil { + log.Printf("ERROR: fetchSettings: %v", err) + } + // TODO: Take into account availability window, reminder frequency and timezone. + // newReminderTime = lastReminder + (aval window size)/Frequency + for chatID, _ := range cs { + const frequency = 1 + rt, err := r.LastReminderTime(chatID) + if err != nil { + log.Print(err) + } + newRT := rt.Add(24 / frequency * time.Hour) + if time.Now().After(newRT) { + if err := r.sendNofication(&Notification{chatID}); err != nil { + log.Print(err) + } + if err := r.UpdateLastReminderTime(chatID); err != nil { + log.Print(err) + } + } + } + select { + case <-ticker: + case <-cancel: + return + } + } +} diff --git a/reminder_test.go b/reminder_test.go new file mode 100644 index 0000000..f639a85 --- /dev/null +++ b/reminder_test.go @@ -0,0 +1,78 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +func TestReminders(t *testing.T) { + dir, err := ioutil.TempDir("", "repetition") + if err != nil { + t.Fatal(err) + } + t.Logf("Temp dir: %q", dir) + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + + settings, err := NewSettingsConfig(dbPath) + if err != nil { + t.Fatal(err) + } + + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + + r, err := NewReminder(&Clients{ + Settings: settings, + }, db) + if err != nil { + t.Fatal(err) + } + + const chatID int64 = 0 + if err := settings.Set(chatID, DefaultSettings()); err != nil { + t.Fatal(err) + } + + c := make(chan time.Time) + + cancel := make(chan struct{}) + go func() { + c <- time.Now() + cancel <- struct{}{} + }() + + var sent []*Notification + r.sendNofication = func(n *Notification) error { + sent = append(sent, n) + return nil + } + + r.Loop(c, cancel) + + if len(sent) != 1 { + t.Errorf("got %d notifications (%v), want 1", len(sent), sent) + } +} diff --git a/settings.go b/settings.go index aa0d91a..3025467 100644 --- a/settings.go +++ b/settings.go @@ -83,6 +83,33 @@ func NewSettingsConfig(dbPath string) (*SettingsConfig, error) { return &SettingsConfig{db}, nil } +func (c *SettingsConfig) GetAll() (map[int64]*Settings, error) { + rows, err := c.db.Query(` + SELECT chat_id, settings + FROM Settings`) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + defer rows.Close() + + r := make(map[int64]*Settings) + + for rows.Next() { + var ( + chatID int64 + s string + ) + if err := rows.Scan(&chatID, &s); err != nil { + return nil, err + } + r[chatID] = SettingsFromString(s) + } + return r, nil +} + func (c *SettingsConfig) Get(chatID int64) (*Settings, error) { row := c.db.QueryRow(` SELECT settings @@ -113,13 +140,13 @@ func (c *SettingsConfig) Set(chatID int64, s *Settings) error { func (c *SettingsConfig) SetLanguage(chatid int64, language string) error { currentSettings, err := c.Get(chatid) if err == nil { - languageSettings, ok := SupportedInputLanguages[language]; - if (!ok) { + languageSettings, ok := SupportedInputLanguages[language] + if !ok { return fmt.Errorf("unsupported language %q", language) } - currentSettings.InputLanguage = languageSettings.InputLanguage; - currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3; - currentSettings.TranslationLanguages = languageSettings.TranslationLanguages; + currentSettings.InputLanguage = languageSettings.InputLanguage + currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3 + currentSettings.TranslationLanguages = languageSettings.TranslationLanguages return c.Set(chatid, currentSettings) } return nil diff --git a/settings_test.go b/settings_test.go index be3a5d0..f020aee 100644 --- a/settings_test.go +++ b/settings_test.go @@ -55,4 +55,15 @@ func TestSettings(t *testing.T) { if !reflect.DeepEqual(s, ns) { t.Errorf("new settings were not set! old: %v\n new: %v", s, ns) } + + gotAll, err := settings.GetAll() + if err != nil { + t.Error(err) + } + wantAll := map[int64]*Settings{ + chatID: ns, + } + if !reflect.DeepEqual(gotAll, wantAll) { + t.Errorf("settings.GetAll() got: %v want: %v", gotAll, wantAll) + } } From a6d864a42f17c87f0620216ea2807d103b903657 Mon Sep 17 00:00:00 2001 From: Attila Date: Sun, 25 Oct 2020 16:36:37 +0000 Subject: [PATCH 4/8] Use more declarative top-level structure. The new top-level structure has several advantages compared to the action-based structure it replaces: 1. It's easier to see what commands are supported just from glancing over at the commands template. 2. There is less duplication in the commands definition, so it should be much easier to add additional commands, and settings. 3. Overall behaviour becomes less statefull. It's annoying to do /stop in the middle of the practive to look up some word from the usage examples. --- actions.go | 695 +--------------------------------------------- callbacks.go | 165 +++++++++++ commands.go | 53 +--- commandsV2.go | 654 +++++++++++++++++++++++++++++++++++++++++++ e2e_test.go | 11 +- settings.go | 45 ++- testdata/e2e.json | 84 +++--- 7 files changed, 912 insertions(+), 795 deletions(-) create mode 100644 callbacks.go create mode 100644 commandsV2.go diff --git a/actions.go b/actions.go index 7b6df0d..6ed8e52 100644 --- a/actions.go +++ b/actions.go @@ -17,24 +17,7 @@ // users can take. package main -import ( - "database/sql" - "fmt" - "log" - "math/rand" - "sort" - "strings" -) - -// Questions: -// * Should actions be exclusive of one another? If one is executed, the rest are skipped? -// * What about calling if !Match in each action to make sure preconditions hold? NO. Difficult chaining to reuse. -type Action interface { - // Perform the action. - Perform(*Update) ([]Action, error) - // Whether this action should be performed as a result of this update. - Match(*Update) bool -} +import "fmt" // TODO: I am not sure if this is the best decision to bundle all up together. // All objects needed to perform actions. @@ -45,368 +28,6 @@ type Clients struct { Settings *SettingsConfig } -// Actions that always can be executed independent of other things. -func BaseActions(c *Clients) []Action { - return []Action{ - &DontKnowAction{c, ""}, - &LearnAction{c}, - &DeleteWordAction{c}, - &CatchAllAction{c, ""}, - } -} - -// actions shared by all commands: -// b:Don't know action (reset) - to work with all messages, no further -// practice to display. -// b:Learn - should always be possible to add another word to the ones learning - -// THese are more for RegularActions or Default actions -func DefaultActions(c *Clients) []Action { - return []Action{ - &StartAction{c}, - &SettingsAction{c}, - &PracticeAction{c}, - &AddCommandAction{c}, - &DefineWordAction{c}, - // /start - // /settings - // /practice - // word - /// catch the rest and show help? - } -} - -// Practice Actions: -// b:Know & display another (If exited practice mode, or word from callback != -// word from action - do nothing) -// b:Don't know & display another (same as b:Know) -// /stop -// /settings -/// -// (may or may not include /start) - -// func NewPracticeActions(c *Clients, asking string) []Action { -// return []Action{ -// &StartAction{}, -// } -// } - -// PracticeKnowAction{asking string} -// PracticeKnowAction.Perform{ ... return NewPracticeState(a.asking) } - -type StartAction struct { - *Clients -} - -// CallbackAction as a special action to handle all callbacks? - -func (a *StartAction) Perform(u *Update) ([]Action, error) { - // It might be beneficial not to reset actions here. - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(u.Message.Chat.Id, - "Welcome to the language bot. Still in development. No instructions "+ - "so far. "+ - "All sentences and translations are from Tatoeba's (https://tatoeba.org) "+ - "dataset, released under a CC-BY 2.0 FR.") -} - -func (*StartAction) Match(u *Update) bool { - // FIXME: This looks refactorable! - if u.Message == nil { - return false - } - return u.Message.Text == "/start" -} - -type StopAction struct { - *Clients - Message string -} - -func (a *StopAction) Perform(u *Update) ([]Action, error) { - msg := a.Message - if msg == "" { - msg = "Stopped." - } - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(u.Message.Chat.Id, - msg) -} - -func (a *StopAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/stop" -} - -// An action to be executed if no other action was matched. -type CatchAllAction struct { - *Clients - Message string -} - -func (a *CatchAllAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - msg := a.Message - if msg == "" { - switch { - case u.Message != nil: - msg += fmt.Sprintf("Couldn't process your message %q", u.Message.Text) - case u.CallbackQuery != nil: - msg += fmt.Sprintf("INTERNAL: Due to restart or a bug couldn't process callback query %v", u.CallbackQuery) - default: - msg += fmt.Sprintf("INTERNAL: Cannot handle update %v", u) - } - } - return nil, a.Telegram.SendTextMessage(chatId, msg) -} - -func (a *CatchAllAction) Match(u *Update) bool { - return true -} - -type CatchAllMessagesAction struct{ CatchAllAction } - -func (a *CatchAllMessagesAction) Match(u *Update) bool { - return u.Message != nil -} - -// FIXME: Should they be "inherited" from Clients? (I think that makes sense) -type SettingsAction struct { - *Clients -} - -func (a *SettingsAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/settings" -} - -func (a *SettingsAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - s, err := a.Settings.Get(chatId) - if err != nil { - return nil, err - } - var ls []string - for l, v := range s.TranslationLanguages { - if v { - ls = append(ls, fmt.Sprintf("%q", l)) - } - } - sort.Strings(ls) - msg := fmt.Sprintf(` -Current settings: - -Input language: %q -Input language in ISO 639-3: %q -Translation languages in ISO 639-3: %s -Time Zone: %s - -Choose setting which you want to modify. -(The choice might improve in the future.) -`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ","), s.TimeZone) - return []Action{ - // Note that stop should be handled before input language is! - &StopAction{a.Clients, "Exited settings"}, - &InputLanguageButton{a.Clients}, - &TimeZoneButton{a.Clients}, - &PracticeAction{a.Clients}, - &CatchAllMessagesAction{CatchAllAction{ - a.Clients, "type /stop to exit settings"}}, - }, a.Telegram.SendMessage(&MessageReply{ - ChatId: chatId, - Text: msg, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{{ - { - Text: "Input Language", - CallbackData: CallbackInfo{ - Action: ChangeSettingAction, - Setting: "InputLanguage", - }.String(), - }, - { - Text: "Time Zone", - CallbackData: CallbackInfo{ - Action: ChangeSettingAction, - Setting: "TimeZone", - }.String(), - }, - }}, - }, - }) -} - -type InputLanguageButton struct { - *Clients -} - -func (b *InputLanguageButton) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - var ls []string - for l, _ := range SupportedInputLanguages { - ls = append(ls, fmt.Sprintf("%q", l)) - } - sort.Strings(ls) - return []Action{ - &StopAction{b.Clients, "Exited settings"}, - &PickLanguageAction{b.Clients}, - }, b.Telegram.SendTextMessage( - chatId, - fmt.Sprintf("Enter input language of your choice. Supported are %s", - strings.Join(ls, ","))) -} - -func (b *InputLanguageButton) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Setting == "InputLanguage" -} - -type TimeZoneButton struct { - *Clients -} - -func (b *TimeZoneButton) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - return []Action{ - &StopAction{b.Clients, "Exited settings"}, - &PickTimeZoneAction{b.Clients}, - }, b.Telegram.SendMessage(&MessageReply{ - ChatId: chatId, - Text: "Input your timezone in format (UTC, UTC+X or UTC-X)", - }) -} - -func (b *TimeZoneButton) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Setting == "TimeZone" -} - -type PickLanguageAction struct { - *Clients -} - -func (a *PickLanguageAction) Perform(u *Update) ([]Action, error) { - m := u.Message - chatId := m.Chat.Id - err := a.Settings.SetLanguage(chatId, m.Text) - if err != nil { - return nil, a.Telegram.SendTextMessage(chatId, "Unsupported language. Try again.") - } - sa := SettingsAction{a.Clients} - return sa.Perform(u) -} - -func (a *PickLanguageAction) Match(u *Update) bool { - return u.Message != nil -} - -type PickTimeZoneAction struct { - *Clients -} - -func (a *PickTimeZoneAction) Perform(u *Update) ([]Action, error) { - m := u.Message - chatId := m.Chat.Id - set := TimeZones[m.Text] - if !set { - return nil, a.Telegram.SendTextMessage(chatId, "Unsupported time zone. Try again (format should be UTC, UTC+X or UTC-X).") - } - a.Settings.SetTimeZone(chatId, m.Text) - sa := SettingsAction{a.Clients} - return sa.Perform(u) -} - -func (a *PickTimeZoneAction) Match(u *Update) bool { - return u.Message != nil -} - -type PracticeAction struct { - *Clients -} - -func (a *PracticeAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - switch UsePractice { - case PracticeWordSpelling: - // FIXME: This is not tested!!! - d, err := a.Repetitions.Repeat(chatId) - if err == sql.ErrNoRows { - // FIXME: This might be refactorable - return DefaultActions(a.Clients), - a.Telegram.SendTextMessage(chatId, "No more rows to practice; exiting practice mode.") - } - if err != nil { - return nil, fmt.Errorf("retrieving word for repetition: %w", err) - } - return []Action{ - &StopAction{a.Clients, "Stopped practice"}, - &SettingsAction{a.Clients}, - &AnswerAction{a.Clients, d}, - // FIXME: Obfuscate definition before sending it to the user!!! - }, a.Telegram.SendTextMessage(chatId, d) - case PracticeKnowledge: - w, err := a.Repetitions.RepeatWord(chatId) - if err == sql.ErrNoRows { - return DefaultActions(a.Clients), - a.Telegram.SendTextMessage(chatId, "No more rows to practice; exiting practice mode.") - } - if err != nil { - return nil, fmt.Errorf("retrieving word for repetition: %w", err) - } - ka := &KnowAction{a.Clients, w} - dka := &DontKnowAction{a.Clients, w} - return []Action{ - &StopAction{a.Clients, "Stopped practice"}, - &SettingsAction{a.Clients}, - ka, - dka, - &CatchAllMessagesAction{CatchAllAction{ - a.Clients, "type /stop to exit practice mode"}}, - }, a.Telegram.SendMessage(&MessageReply{ - ChatId: chatId, - Text: w, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - ka.AsKeyboard(), - dka.AsKeyboard(), - }}, - }, - }) - default: - return nil, fmt.Errorf("INTERNAL: Unimplemented practice type: %v", UsePractice) - } -} - -func (a *PracticeAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return u.Message.Text == "/practice" -} - // TODO: Can I not extract word from the message? m.Text? func flipWordCard(c *Clients, word string, m *Message, ks []*InlineKeyboard) error { // TODO: It isn't always neccessary to retrieve defitnion when this @@ -436,317 +57,3 @@ func flipWordCard(c *Clients, word string, m *Message, ks []*InlineKeyboard) err } return nil } - -type KnowAction struct { - *Clients - Word string -} - -func (a *KnowAction) Perform(u *Update) ([]Action, error) { - defer a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, "") - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.AnswerKnow(chatId, word); err != nil { - return nil, err - } - dka := &DontKnowAction{a.Clients, a.Word} - if err := flipWordCard(a.Clients, word, u.CallbackQuery.Message, []*InlineKeyboard{dka.AsKeyboard()}); err != nil { - return nil, err - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *KnowAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == PracticeKnowAction && info.Word == a.Word -} - -func (a *KnowAction) AsKeyboard() *InlineKeyboard { - return &InlineKeyboard{ - Text: "Know", - CallbackData: CallbackInfo{ - Action: PracticeKnowAction, - Word: a.Word, - }.String(), - } -} - -type DontKnowAction struct { - *Clients - Word string -} - -func (a *DontKnowAction) Perform(u *Update) ([]Action, error) { - defer a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, "Reset progress") - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.AnswerDontKnow(chatId, word); err != nil { - return nil, err - } - if err := flipWordCard(a.Clients, word, u.CallbackQuery.Message, nil); err != nil { - return nil, err - } - // Clicking Don't know on the previous messages shouldn't result in - // practice being continued. - if word != a.Word { - return nil, nil - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *DontKnowAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == PracticeDontKnowAction -} - -func (a *DontKnowAction) AsKeyboard() *InlineKeyboard { - return &InlineKeyboard{ - Text: "Don't know", - CallbackData: CallbackInfo{ - Action: PracticeDontKnowAction, - Word: a.Word, - }.String(), - } -} - -type AnswerAction struct { - *Clients - Question string -} - -func (a *AnswerAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - m := u.Message - correct, err := a.Repetitions.Answer(chatId, a.Question, m.Text) - if err != nil { - return nil, err - } - if correct != m.Text { - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Correct word: %q", correct)) - } - // correct == m.Text - congrats := [...]string{"Good job!", "Well done!", "Awesome!", "Keep it up!", "Excellent!", "You've got it!", "Great!", "Terrific!", "Ваще красава!11"} - if err := a.Telegram.SendTextMessage(chatId, congrats[rand.Int()%len(congrats)]); err != nil { - return nil, err - } - pa := PracticeAction{a.Clients} - return pa.Perform(u) -} - -func (a *AnswerAction) Match(u *Update) bool { - return u.Message != nil -} - -type AddCommandAction struct { - *Clients -} - -func (a *AddCommandAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - return []Action{ - &StopAction{a.Clients, "Cancelled addition"}, - &SettingsAction{a.Clients}, - &AddWordAction{a.Clients}, - }, a.Telegram.SendTextMessage(chatId, ` -Enter the card you want to add in the format: - - -`) -} - -func (a *AddCommandAction) Match(u *Update) bool { - return u.Message != nil && u.Message.Text == "/add" -} - -type AddWordAction struct { - *Clients -} - -func (a *AddWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - parts := strings.Split(u.Message.Text, "\n") - if len(parts) < 2 { - return nil, a.Telegram.SendTextMessage(chatId, "Wrong format") - } - front := parts[0] - back := strings.Join(parts[1:], "\n") - if err := a.Repetitions.Save(chatId, front, back); err != nil { - return nil, err - } - return DefaultActions(a.Clients), a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Added %q for learning!", front)) -} - -func (a *AddWordAction) Match(u *Update) bool { - return u.Message != nil -} - -type DefineWordAction struct { - *Clients -} - -func (a *DefineWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - m := u.Message - if len(strings.Split(m.Text, " ")) > 1 { - return nil, a.Telegram.SendTextMessage(chatId, "INTERNAL ERROR: DefineWordAction should not have been called with many words. Is Match called correctly?") - } - - def, err := a.Repetitions.GetDefinition(m.Chat.Id, m.Text) - if err == nil { - dka := &DontKnowAction{a.Clients, m.Text} - k := dka.AsKeyboard() - k.Text = "Reset progress" - return nil, a.Telegram.SendMessage(&MessageReply{ - ChatId: m.Chat.Id, - Text: def, - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{ - []*InlineKeyboard{k}, - }, - }, - }) - } - if err != sql.ErrNoRows { - log.Printf("ERROR: Repetitions(%d, %s): %v", m.Chat.Id, m.Text, err) - } - settings, err := a.Settings.Get(chatId) - if err != nil { - return nil, fmt.Errorf("get settings: %v", err) - } - ds, err := a.Definer.Define(m.Text, settings) - if err != nil { - // TODO: Might be good to post debug logs to the reply in the debug mode. - log.Printf("Error fetching the definition: %w", err) - // FIXME: Add search url to the reply? - return nil, a.Telegram.SendTextMessage(chatId, "Couldn't find definitions") - } - - for _, d := range ds { - if err := a.Telegram.SendMessage(&MessageReply{ - ChatId: m.Chat.Id, - Text: d, - ParseMode: "MarkdownV2", - ReplyMarkup: &ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ - &InlineKeyboard{ - Text: "Learn", - CallbackData: CallbackInfo{ - Action: SaveWordAction, - Word: m.Text, - }.String(), - }, - }}, - }, - }); err != nil { - return nil, err - } - } - return nil, nil -} - -func (a *DefineWordAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return len(strings.Split(u.Message.Text, " ")) == 1 -} - -type LearnAction struct { - *Clients -} - -func (a *LearnAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - word := info.Word - if err := a.Repetitions.Save(chatId, word, u.CallbackQuery.Message.Text); err != nil { - return nil, err - } - m := u.CallbackQuery.Message - r := &EditMessageText{ - ChatId: m.Chat.Id, - MessageId: m.Id, - ReplyMarkup: ReplyMarkup{ - InlineKeyboard: [][]*InlineKeyboard{ - []*InlineKeyboard{}, - }, - }, - } - var rm Message - if err := a.Telegram.Call("editMessageReplyMarkup", r, &rm); err != nil { - return nil, fmt.Errorf("editing message reply markup: %w", err) - } - msg := fmt.Sprintf("Saved %q for learning", word) - a.Telegram.AnswerCallbackLog(u.CallbackQuery.Id, msg) - return nil, nil -} - -func (a *LearnAction) Match(u *Update) bool { - if u.CallbackQuery == nil { - return false - } - info := CallbackInfoFromString(u.CallbackQuery.Data) - return info.Action == SaveWordAction -} - -// DeleteWordAction stops learning of the word. -type DeleteWordAction struct { - *Clients -} - -func (a *DeleteWordAction) Match(u *Update) bool { - if u.Message == nil { - return false - } - return strings.HasPrefix(u.Message.Text, "/delete") -} - -func (a *DeleteWordAction) Perform(u *Update) ([]Action, error) { - chatId, err := u.ChatId() - if err != nil { - return nil, err - } - word := strings.TrimSpace(strings.TrimPrefix(u.Message.Text, "/delete")) - e, err := a.Repetitions.Exists(chatId, word) - if err != nil { - return nil, err - } - if !e { - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Word %q isn't saved for learning!", word)) - } - if err := a.Repetitions.Delete(chatId, word); err != nil { - return nil, err - } - return nil, a.Telegram.SendTextMessage(chatId, fmt.Sprintf("Deleted %q!", word)) -} diff --git a/callbacks.go b/callbacks.go new file mode 100644 index 0000000..d2d33da --- /dev/null +++ b/callbacks.go @@ -0,0 +1,165 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +type KnowCallback struct { + Word string +} + +func (KnowCallback) Call(s *State, q *CallbackQuery) error { + defer s.Telegram.AnswerCallbackLog(q.Id, "") + chatID := q.Message.Chat.Id + word := CallbackInfoFromString(q.Data).Word + + // TODO: Need to handle 2 rapid taps to avoid saving it as known 2 times in a row. + if err := s.Repetitions.AnswerKnow(chatID, word); err != nil { + return err + } + + if err := flipWordCard(s.Clients, word, q.Message, []*InlineKeyboard{DontKnowCallback{word, false}.AsInlineKeyboard()}); err != nil { + return err + } + return practiceReply(s, chatID) +} + +func (KnowCallback) Match(s *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == PracticeKnowAction +} + +func (k KnowCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Know", + CallbackData: CallbackInfo{ + Action: PracticeKnowAction, + Word: k.Word, + }.String(), + } +} + +type DontKnowCallback struct { + Word string + // If true when clicking another practice card will be shown. + Practice bool +} + +func (DontKnowCallback) Call(s *State, q *CallbackQuery) error { + defer s.Telegram.AnswerCallbackLog(q.Id, "Reset progress") + info := CallbackInfoFromString(q.Data) + chatID := q.Message.Chat.Id + word := info.Word + + if err := s.Repetitions.AnswerDontKnow(chatID, word); err != nil { + return err + } + + if err := flipWordCard(s.Clients, word, q.Message, nil); err != nil { + return err + } + + if info.Action == PracticeDontKnowActionNoPractice { + return nil + } + return practiceReply(s, chatID) +} + +func (DontKnowCallback) Match(_ *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == PracticeDontKnowAction || info.Action == PracticeDontKnowActionNoPractice +} + +func (c DontKnowCallback) AsInlineKeyboard() *InlineKeyboard { + a := PracticeDontKnowActionNoPractice + if c.Practice { + a = PracticeDontKnowAction + } + return &InlineKeyboard{ + Text: "Don't know", + CallbackData: CallbackInfo{ + Action: a, + Word: c.Word, + }.String(), + } +} + +type ResetProgressCallback struct { + Word string +} + +// ResetProgress type is just a convenience placeholder to create inline keyboards. +// Logic will be handled by dont know callback. +func (ResetProgressCallback) Call(_ *State, _ *CallbackQuery) error { + return nil +} + +func (ResetProgressCallback) Match(_ *State, _ *CallbackQuery) bool { + return false +} + +func (c ResetProgressCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Reset progress", + CallbackData: CallbackInfo{ + Action: PracticeDontKnowActionNoPractice, + Word: c.Word, + }.String(), + } +} + +type LearnCallback struct { + Word string +} + +func (LearnCallback) Call(s *State, q *CallbackQuery) error { + // FIXME: Next 3 lines are very common. + chatID := q.Message.Chat.Id + word := CallbackInfoFromString(q.Data).Word + if err := s.Repetitions.Save(chatID, word, q.Message.Text); err != nil { + return err + } + m := q.Message + r := &EditMessageText{ + ChatId: m.Chat.Id, + MessageId: m.Id, + ReplyMarkup: ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{ + []*InlineKeyboard{}, + }, + }, + } + var rm Message + if err := s.Telegram.Call("editMessageReplyMarkup", r, &rm); err != nil { + return fmt.Errorf("editing message reply markup: %w", err) + } + msg := fmt.Sprintf("Saved %q for learning", word) + s.Telegram.AnswerCallbackLog(q.Id, msg) + return nil +} + +func (LearnCallback) Match(_ *State, q *CallbackQuery) bool { + info := CallbackInfoFromString(q.Data) + return info.Action == SaveWordAction +} + +func (c LearnCallback) AsInlineKeyboard() *InlineKeyboard { + return &InlineKeyboard{ + Text: "Learn", + CallbackData: CallbackInfo{ + Action: SaveWordAction, + Word: c.Word, + }.String(), + } +} diff --git a/commands.go b/commands.go index 9a9a321..5f7e227 100644 --- a/commands.go +++ b/commands.go @@ -42,11 +42,6 @@ const ( const UsePractice = PracticeKnowledge var ( - SupportedInputLanguages map[string]Settings - TimeZones map[string]bool -) - -func init() { SupportedInputLanguages = map[string]Settings{ "Hungarian": Settings{ InputLanguage: "Hungarian", @@ -83,7 +78,7 @@ func init() { timeZones["UTC"] = true return timeZones }() -} +) type CallbackAction int @@ -91,7 +86,7 @@ const ( SaveWordAction CallbackAction = iota PracticeKnowAction PracticeDontKnowAction - ChangeSettingAction + PracticeDontKnowActionNoPractice ) // Make sure all fields are Public, otherwise encoding will not work @@ -124,10 +119,7 @@ func (c CallbackInfo) String() string { type Commander struct { *Clients - // chat_id -> Available Actions - actions map[int64][]Action - // Actions that should be always available. - baseActions []Action + bot *Bot } type CommanderOptions struct { @@ -211,48 +203,25 @@ func NewCommander(tm *Telegram, opts *CommanderOptions) (*Commander, error) { log.Printf("getMe: %s", string(raw)) return &Commander{ - Clients: c, - actions: make(map[int64][]Action), - baseActions: BaseActions(c), + Clients: c, + bot: &Bot{ + state: &State{c}, + command: make(map[int64]Command), + }, }, nil } -func (c *Commander) Actions(chatId int64) []Action { - a, ok := c.actions[chatId] - if !ok { - a = DefaultActions(c.Clients) - } - return append(a, c.baseActions...) -} - // Update processes the user's update and spit out output. // Should return an error only on unrecoverable errors due to which we cannot // continue execution. // TODO: Use answerCallbackQuery to notify client that callback was processed? func (c *Commander) Update(u *Update) error { - chatId, err := u.ChatId() + err := c.bot.Update(u) if err != nil { // Not sure what to do otherwise, but crashing isn't nice. - log.Printf("INTENAL ERROR: {%+v}.ChatId(): %v", u, err) - return nil + log.Printf("INTENAL ERROR for update %v: %v", u, err) } - for _, a := range c.Actions(chatId) { - if a.Match(u) { - na, err := a.Perform(u) - // FIXME: on error maybe some action changes are warranted? - if err != nil { - // FIXME: Don't return an error here!!! Display it. - return err - } - // FIXME seems annoying to keep track of current actions when on wron - // input the set of actions should never change. - if len(na) > 0 { - c.actions[chatId] = na - } - return nil - } - } - return fmt.Errorf("Did not process an update: %v!", u.CallbackQuery) + return nil } func (c *Commander) PollAndProcess() error { diff --git a/commandsV2.go b/commandsV2.go new file mode 100644 index 0000000..9f0275c --- /dev/null +++ b/commandsV2.go @@ -0,0 +1,654 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// commandsV2 is a replacement for action-based message processing. Everything +// inside here is responsible for tying database together with telegram +// interactions. +package main + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "sort" + "strings" +) + +type Callback interface { + Call(*State, *CallbackQuery) error + Match(*State, *CallbackQuery) bool + AsInlineKeyboard() *InlineKeyboard +} + +type State struct { + *Clients +} + +// TODO: +func (s *State) LoadCommand(chatID int64) (*SerializedCommand, error) { + return nil, nil +} + +// TODO: +func (s *State) SaveCommand(chatID int64, _ *SerializedCommand) error { + return nil +} + +type Bot struct { + state *State + command map[int64]Command +} + +func (b *Bot) fetchCommand(chatID int64) Command { + cmd := b.command[chatID] + if cmd == nil { + s, err := b.state.LoadCommand(chatID) + if err == nil { + cmd, err = s.AsCommand() + } + if err != nil { + cmd = nil + log.Printf("INTERNAL ERROR: couldn't fetch command for chat %d: %v", chatID, err) + } + if cmd == nil { + cmd = CommandsTemplate.DefaultCommand("") + } + } + b.command[chatID] = cmd + return cmd +} + +func (b *Bot) updateCommand(chatID int64, cmd Command) error { + b.command[chatID] = cmd + var s *SerializedCommand + if cmd != nil { + s = cmd.Serialize() + } + return b.state.SaveCommand(chatID, s) +} + +func (b *Bot) Update(u *Update) (err error) { + chatId, _ := u.ChatId() + if err != nil { + return err + } + + // Try surfacing UserError and update the bot accordingly on internal error. + defer func() { + var e UserError + if errors.As(err, &e) { + err = e.Surface(b.state) + } + if err == nil { + return + } + log.Printf("INTERNAL ERROR: %v", err) + // Reset command so that user wouldn't be stuck with internal errors + // with no way out. + if upErr := b.updateCommand(chatId, nil); upErr != nil { + log.Printf("INTERNAL ERROR: updateCommand(%d, nil): %v; while handling %v", chatId, upErr, err) + } + }() + + if u.CallbackQuery != nil { + for _, c := range CommandsTemplate.Callbacks { + if c.Match(b.state, u.CallbackQuery) { + return c.Call(b.state, u.CallbackQuery) + } + } + return fmt.Errorf("INTERNAL ERROR: Did find a corresponding callback for callback query: %v", u.CallbackQuery) + } + + if u.Message == nil { + // TODO: Make internal error type. + return fmt.Errorf("INTERNAL ERROR: Update is neither a message, nor a callback query: %v", u) + } + + // Update is a Message. + msg := u.Message.Text + for n, f := range CommandsTemplate.Commands { + if msg == n { + cmd := f(n) + cmd, err = cmd.OnCommand(b.state, u.Message) + if err != nil { + return err + } + if cmd == nil { + cmd = CommandsTemplate.DefaultCommand("") + } + return b.updateCommand(chatId, cmd) + } + } + + // None of the commands match, so process the message. + cmd := b.fetchCommand(chatId) + cmd, err = cmd.ProcessMessage(b.state, u.Message) + // On user caused error command should still be updated accordingly. + if err == nil || errors.Is(err, UserError{}) { + return b.updateCommand(chatId, cmd) + } + return err +} + +type SerializedCommand struct { + // Name of the command + Name string + // Serialized command's state. Command should be able to restore it's state + // from this string. + Data []byte +} + +func (s *SerializedCommand) AsCommand() (Command, error) { + factory := CommandsTemplate.DefaultCommand + if s == nil { + return factory(""), nil + } + for n, f := range CommandsTemplate.Commands { + if n == s.Name { + factory = f + } + } + cmd := factory(s.Name) + return cmd, cmd.Init(s) +} + +type Command interface { + Serialize() *SerializedCommand + Init(*SerializedCommand) error + // Process message from the user. + ProcessMessage(*State, *Message) (Command, error) + // Called when user runs this command. + OnCommand(*State, *Message) (Command, error) + // Both OnCommand and ProcessMessage return the new context command +} + +// Factory is used so that Command metadata is not changed accidentally. +type CommandFactory func(name string) Command + +// UserError is an error that should be surfaced to the user. +type UserError struct { + Err error + ChatID int64 +} + +func (u UserError) Error() string { + return u.Err.Error() +} + +// Surface surfaces error to the user. +func (u UserError) Surface(s *State) error { + e := u.Error() + if len(e) > 0 { + e = strings.ToUpper(e[:1]) + e[1:] + } + return s.Telegram.SendTextMessage(u.ChatID, e) +} + +type question struct { + name string + ask func(s *State, chatID int64) error + validate func(*State, *Message) error + answer string +} + +type multiQuestionCommand struct { + name string + // All questions should have unique names. + // range is used instead of map so that the called can control questions' order. + questions []*question + // Once user answers all questions save will be called. questions will have + // answers populated. + save func(state *State, chatID int64, questions []*question) error + // Name of the last question we have asked. + lastQuestion string +} + +func MultiQuestionCommandFactory(questions []*question, save func(state *State, chatID int64, questions []*question) error) CommandFactory { + return func(name string) Command { + return &multiQuestionCommand{ + name: name, + questions: questions, + save: save, + } + } +} + +type multiQuestionCommandSerialized struct { + answers map[string]string + lastQuestion string +} + +func (c *multiQuestionCommand) Serialize() *SerializedCommand { + // No need to serialize question names, they should be the same in CommandsTemplate. + // Need to serialize answers to the questions though. + a := make(map[string]string) + for _, q := range c.questions { + a[q.name] = q.answer + } + cs := &multiQuestionCommandSerialized{ + answers: a, + lastQuestion: c.lastQuestion, + } + b, err := json.Marshal(cs) + if err != nil { + log.Printf("INTERNAL ERROR: Couldn't serialize %v: %v", cs, err) + } + return &SerializedCommand{ + Name: c.name, + Data: b, + } +} + +func (c *multiQuestionCommand) Init(s *SerializedCommand) error { + cs := &multiQuestionCommandSerialized{} + if err := json.Unmarshal(s.Data, &cs); err != nil { + return fmt.Errorf("Unmarshal(%s): %w", s.Data, err) + } + for _, q := range c.questions { + q.answer = cs.answers[q.name] + } + c.lastQuestion = cs.lastQuestion + return nil +} + +func (c *multiQuestionCommand) OnCommand(s *State, m *Message) (Command, error) { + chatID := m.Chat.Id + if len(c.questions) == 0 { + // With to questions, no reason to have this command process messages. + return nil, c.save(s, chatID, c.questions) + } + if err := c.questions[0].ask(s, chatID); err != nil { + return nil, err + } + c.lastQuestion = c.questions[0].name + return c, nil +} + +func (c *multiQuestionCommand) ProcessMessage(s *State, m *Message) (Command, error) { + var q *question + for _, qe := range c.questions { + if qe.name == c.lastQuestion { + q = qe + break + } + } + if q == nil { + return nil, fmt.Errorf("INTERNAL ERROR: Did not find a question corresponding to last question %s", c.lastQuestion) + } + if err := q.validate(s, m); err != nil { + // In case validate fails with user error, we want to be able to retry, + // so we return c to be a new command. + return c, err + } + q.answer = m.Text + + var next *question = nil + for _, qe := range c.questions { + if qe.answer == "" { + next = qe + break + } + } + if next == nil { + err := c.save(s, m.Chat.Id, c.questions) + // After all questions have been answered there is no point in + // keeping trying to save, even if it fails with UserError. + return nil, err + } + if err := next.ask(s, m.Chat.Id); err != nil { + // ask should never fail with user error. + return nil, err + } + c.lastQuestion = next.name + return c, nil +} +func ReplyCommand(reply func(s *State, chatID int64) error) CommandFactory { + return MultiQuestionCommandFactory( + nil, + func(s *State, chatID int64, _ []*question) error { + return reply(s, chatID) + }, + ) +} + +func NewMessageReply(chatID int64, text string, callbacks []Callback) *MessageReply { + var ik []*InlineKeyboard + for _, c := range callbacks { + ik = append(ik, c.AsInlineKeyboard()) + } + var rm *ReplyMarkup + if len(ik) > 0 { + rm = &ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{ik}, + } + } + return &MessageReply{ + ChatId: chatID, + Text: text, + ReplyMarkup: rm, + } +} + +// practiceReply sends practice card to the user. +func practiceReply(s *State, chatID int64) error { + switch UsePractice { + case PracticeKnowledge: + default: + panic(fmt.Sprintf("INTERNAL: Unimplemented practice type: %v", UsePractice)) + } + word, err := s.Repetitions.RepeatWord(chatID) + if err == sql.ErrNoRows { + // FIXME: Make this user error instead. + return s.Telegram.SendTextMessage(chatID, "No more rows to practice; exiting practice mode.") + } + if err != nil { + return fmt.Errorf("retrieving word for repetition: %w", err) + } + return s.Telegram.SendMessage(NewMessageReply(chatID, word, []Callback{KnowCallback{word}, DontKnowCallback{word, true}})) +} + +// settingsReply sends current settings and instructions on how to change them. +func settingsReply(state *State, chatID int64) error { + s, err := state.Settings.Get(chatID) + if err != nil { + return err + } + var ls []string + for l, v := range s.TranslationLanguages { + if v { + ls = append(ls, fmt.Sprintf("%q", l)) + } + } + sort.Strings(ls) + var cmds []string + for k, _ := range SettingsCommands { + cmds = append(cmds, " "+k) + } + sort.Strings(cmds) + msg := fmt.Sprintf(` +Current settings: + +Input language: %q +Input language in ISO 639-3: %q +Translation languages in ISO 639-3: %s +Time Zone: %s + +To modify settings use one of the commands below: +%s +`, s.InputLanguage, s.InputLanguageISO639_3, strings.Join(ls, ","), s.TimeZone, strings.Join(cmds, "\n")) + return state.Telegram.SendMessage(NewMessageReply(chatID, msg, nil)) +} + +// This inteface is a bit redundant. We need it though to avoid initialization +// loop with SettingsCommands depending on settingsReply and settingsReply +// depending on SettingsCommands. +type SimpleQuestionCommand interface { + Ask(_ *State, chatID int64) error + Validate(*State, *Message) error + Save(_ *State, chatID int64, answer string) error +} + +func SimpleQuestionCommandFactory(c SimpleQuestionCommand) CommandFactory { + return MultiQuestionCommandFactory( + []*question{{ + name: "question", + ask: c.Ask, + validate: c.Validate, + }}, + func(s *State, chatID int64, questions []*question) error { + return c.Save(s, chatID, questions[0].answer) + }, + ) +} + +type SimpleSettingCommand struct { + question string + validate func(s *State, answer string) error + save func(s *State, chatID int64, answer string) error +} + +func (c *SimpleSettingCommand) Ask(s *State, chatID int64) error { + return askQuestion(c.question)(s, chatID) +} + +func (c *SimpleSettingCommand) Validate(s *State, m *Message) error { + if err := c.validate(s, m.Text); err != nil { + return UserError{ChatID: m.Chat.Id, Err: fmt.Errorf("%w. Please try again.", err)} + } + return nil +} + +func (c *SimpleSettingCommand) Save(s *State, chatID int64, answer string) error { + if err := c.save(s, chatID, answer); err != nil { + return err + } + return settingsReply(s, chatID) +} + +func askQuestion(q string) func(s *State, chatID int64) error { + return func(s *State, chatID int64) error { + return s.Telegram.SendTextMessage(chatID, q) + } +} + +func AddCommandFactory() CommandFactory { + noopValidate := func(*State, *Message) error { return nil } + return MultiQuestionCommandFactory( + []*question{{ + name: "front", + ask: askQuestion("Enter front of the card (word, expression, question)."), + validate: noopValidate, + }, { + name: "back", + ask: askQuestion("Enter back of the card (definition, answer)."), + validate: noopValidate, + }}, + func(s *State, chatID int64, qs []*question) error { + var front string + var back string + for _, q := range qs { + switch q.name { + case "front": + front = q.answer + case "back": + back = q.answer + default: + return fmt.Errorf("unexpected question in save: %v", q) + } + } + if err := s.Repetitions.Save(chatID, front, back); err != nil { + return err + } + return s.Telegram.SendTextMessage(chatID, fmt.Sprintf("Added %q for learning!", front)) + }, + ) +} + +func DeleteCommandFactory() CommandFactory { + return MultiQuestionCommandFactory( + []*question{{ + name: "word", + ask: askQuestion("Enter the word you want to delete from learning!"), + validate: func(s *State, m *Message) error { + e, err := s.Repetitions.Exists(m.Chat.Id, m.Text) + if err != nil { + return err + } + if !e { + return UserError{ChatID: m.Chat.Id, Err: fmt.Errorf("Word %q isn't saved for learning!", m.Text)} + } + return nil + }, + }}, + func(s *State, chatID int64, qs []*question) error { + if err := s.Repetitions.Delete(chatID, qs[0].answer); err != nil { + return err + } + return s.Telegram.SendTextMessage(chatID, fmt.Sprintf("Deleted %q!", qs[0].answer)) + }, + ) +} + +type defaultCommand struct{} + +func (defaultCommand) Serialize() *SerializedCommand { + return nil +} +func (defaultCommand) Init(*SerializedCommand) error { + return nil +} +func (defaultCommand) ProcessMessage(s *State, m *Message) (Command, error) { + chatID := m.Chat.Id + + if len(strings.Split(m.Text, " ")) > 1 { + return nil, UserError{ChatID: chatID, Err: fmt.Errorf("For now this bot doesn't work with expressions. Try entering a single work without spaces.")} + } + + def, err := s.Repetitions.GetDefinition(m.Chat.Id, m.Text) + if err == nil { + return nil, s.Telegram.SendMessage(NewMessageReply( + m.Chat.Id, def, + []Callback{ResetProgressCallback{m.Text}})) + } + if err != sql.ErrNoRows { + log.Printf("ERROR: Repetitions(%d, %s): %v", m.Chat.Id, m.Text, err) + } + settings, err := s.Settings.Get(chatID) + if err != nil { + return nil, fmt.Errorf("get settings: %v", err) + } + ds, err := s.Definer.Define(m.Text, settings) + if err != nil { + // TODO: Might be good to post debug logs to the reply in the debug mode. + log.Printf("Error fetching the definition: %w", err) + // TODO: Add search url to the reply? + return nil, UserError{ + ChatID: m.Chat.Id, + Err: fmt.Errorf("Couldn't find definitions."), + } + } + for _, d := range ds { + if err := s.Telegram.SendMessage(&MessageReply{ + ChatId: m.Chat.Id, + Text: d, + ParseMode: "MarkdownV2", + ReplyMarkup: &ReplyMarkup{ + InlineKeyboard: [][]*InlineKeyboard{[]*InlineKeyboard{ + LearnCallback{m.Text}.AsInlineKeyboard(), + }}, + }, + }); err != nil { + return nil, err + } + } + return nil, nil +} + +// Should never be called. +func (defaultCommand) OnCommand(*State, *Message) (Command, error) { + return nil, nil +} + +func joinCommands(m1, m2 map[string]CommandFactory) map[string]CommandFactory { + r := make(map[string]CommandFactory) + for k, v := range m1 { + if _, dup := m2[k]; dup { + panic(fmt.Sprintf("Found duplicated command: %q", k)) + } + r[k] = v + } + for k, v := range m2 { + // There can be no additional duplications so no need to check it. + r[k] = v + } + return r +} + +// textReply sends a message and resets state. +func textReply(text string) CommandFactory { + return ReplyCommand(func(s *State, chatID int64) error { + return s.Telegram.SendTextMessage(chatID, text) + }) +} + +// SettingsCommands contains all settings-related commands. They are bundled +// together for convenience to have everything in one place. +var SettingsCommands = map[string]CommandFactory{ + "/language": SimpleQuestionCommandFactory(&SimpleSettingCommand{ + question: func() string { + var ls []string + for l, _ := range SupportedInputLanguages { + ls = append(ls, fmt.Sprintf("%q", l)) + } + sort.Strings(ls) + return fmt.Sprintf("Enter input language of your choice. Supported are %s", + strings.Join(ls, ",")) + }(), + validate: func(s *State, answer string) error { + return s.Settings.ValidateLanguage(answer) + }, + save: func(s *State, chatID int64, answer string) error { + return s.Settings.SetLanguage(chatID, answer) + }, + }), + "/timezone": SimpleQuestionCommandFactory(&SimpleSettingCommand{ + question: "Input your timezone in one of the formats: UTC, UTC+X or UTC-X.", + validate: func(s *State, answer string) error { + return s.Settings.ValidateTimeZone(answer) + }, + save: func(s *State, chatID int64, answer string) error { + return s.Settings.SetTimeZone(chatID, answer) + }, + }), +} + +var CommandsTemplate = struct { + // When receiving a callback it will be matched here. + // Tests should test that each callback is reachable. + // Callbacks can rely only on info got from the CallbackQuery, + // Everything stored in callback should be used only in AsInlineKeyboard method. + Callbacks []Callback + // Each key in the map should directly correspond to the name of the command. + // Possible improvement would be to allow regexps in the Commands name so + // arguments can be passed there. + Commands map[string]CommandFactory + // Command returned by DefaultCommand shouldn't implement OnCommand. It + // should not be ever called. + DefaultCommand CommandFactory +}{ + Commands: joinCommands( + map[string]CommandFactory{ + "/start": textReply( + "Welcome to the language bot. Still in development. No instructions " + + "so far. " + + "All sentences and translations are from Tatoeba's (https://tatoeba.org) " + + "dataset, released under a CC-BY 2.0 FR."), + "/stop": textReply("Stopped. Input the word to get it's definition."), + "/practice": ReplyCommand(practiceReply), + "/settings": ReplyCommand(settingsReply), + "/add": AddCommandFactory(), + "/delete": DeleteCommandFactory(), + }, + SettingsCommands, + ), + Callbacks: []Callback{ + KnowCallback{}, + DontKnowCallback{}, + LearnCallback{}, + }, + DefaultCommand: func(string) Command { return defaultCommand{} }, +} diff --git a/e2e_test.go b/e2e_test.go index c98bbd8..1e5d083 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -87,7 +87,7 @@ b:Don't know /settings -b:Input Language +/language NotARealLanguage @@ -97,15 +97,17 @@ English /settings -b:Input Language +/language Hungarian -b:Input Language +/language /stop -/delete falu +/delete + +falu /practice @@ -116,6 +118,7 @@ b:Input Language /add cardfront + cardback (definitions or what not) cardfront diff --git a/settings.go b/settings.go index 3025467..a099a01 100644 --- a/settings.go +++ b/settings.go @@ -137,26 +137,45 @@ func (c *SettingsConfig) Set(chatID int64, s *Settings) error { return nil } +func (c *SettingsConfig) ValidateLanguage(language string) error { + _, ok := SupportedInputLanguages[language] + if !ok { + return fmt.Errorf("unsupported language %q", language) + } + return nil +} + func (c *SettingsConfig) SetLanguage(chatid int64, language string) error { currentSettings, err := c.Get(chatid) - if err == nil { - languageSettings, ok := SupportedInputLanguages[language] - if !ok { - return fmt.Errorf("unsupported language %q", language) - } - currentSettings.InputLanguage = languageSettings.InputLanguage - currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3 - currentSettings.TranslationLanguages = languageSettings.TranslationLanguages - return c.Set(chatid, currentSettings) + if err != nil { + return err + } + if err := c.ValidateLanguage(language); err != nil { + return err + } + languageSettings := SupportedInputLanguages[language] + currentSettings.InputLanguage = languageSettings.InputLanguage + currentSettings.InputLanguageISO639_3 = languageSettings.InputLanguageISO639_3 + currentSettings.TranslationLanguages = languageSettings.TranslationLanguages + return c.Set(chatid, currentSettings) +} + +func (c *SettingsConfig) ValidateTimeZone(tz string) error { + set := TimeZones[tz] + if !set { + return fmt.Errorf("unsupported time zone (format should be UTC, UTC+X or UTC-X)") } return nil } func (c *SettingsConfig) SetTimeZone(chatid int64, tz string) error { + if err := c.ValidateTimeZone(tz); err != nil { + return err + } currentSettings, err := c.Get(chatid) - if err == nil { - currentSettings.TimeZone = tz - return c.Set(chatid, currentSettings) + if err != nil { + return err } - return nil + currentSettings.TimeZone = tz + return c.Set(chatid, currentSettings) } diff --git a/testdata/e2e.json b/testdata/e2e.json index 5b71ee1..ef7d702 100644 --- a/testdata/e2e.json +++ b/testdata/e2e.json @@ -6,12 +6,12 @@ }, { "Send": "many words", - "Want": "Couldn't process your message \"many words\"", + "Want": "For now this bot doesn't work with expressions. Try entering a single work without spaces.", "WantButtons": null }, { "Send": "oijasdki#noresults#", - "Want": "Couldn't find definitions", + "Want": "Couldn't find definitions.", "WantButtons": null }, { @@ -21,14 +21,14 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", "WantButtons": [ "Learn" ] }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", "WantButtons": [ "Learn" ] @@ -40,7 +40,7 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figuratively\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figuratively\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figuratively, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", "WantButtons": [ "Reset progress" ] @@ -63,10 +63,8 @@ }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { "Send": "/practice", @@ -91,7 +89,7 @@ }, { "Send": "falu", - "Want": "*falu*\n\n1\\. \\[*noun*\\] village\n\nUsage examples:\n\n1\\. Minden falu egy személyt delegálhatott\\.\n\n2\\. Mindegyik falu szép volt\\.\n\n3\\. A gátépítés előtt egy falu volt itt\\.", + "Want": "*falu*\n\n1\\. \\[*noun*\\] village\nA világ egy falu\\.The world is a village\\.\nSynonym: község\nHypernyms: település, helység\nHyponyms: törpefalu \\(\u003c100\\), aprófalu \\(100–500\\), kisfalu \\(500–1,000\\), középfalu \\(1,000–2,000\\), nagyfalu \\(2,000–5,000\\), óriásfalu \\(5,000–10,000 of population\\)\nCoordinate term: város \\(town or city\\)\n\nUsage examples:\n\n1\\. Minden falu egy személyt delegálhatott\\.\n\n2\\. Mindegyik falu szép volt\\.\n\n3\\. A gátépítés előtt egy falu volt itt\\.", "WantButtons": [ "Learn" ] @@ -119,69 +117,66 @@ }, { "Send": "/stop", - "Want": "Stopped practice", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "NotARealLanguage", - "Want": "Unsupported language. Try again.", + "Want": "Unsupported language \"NotARealLanguage\". Please try again.", "WantButtons": null }, { "Send": "English", - "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { "Send": "/stop", - "Want": "Exited settings", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/settings", - "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"English\"\nInput language in ISO 639-3: \"eng\"\nTranslation languages in ISO 639-3: \"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "Hungarian", - "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\n\nChoose setting which you want to modify.\n(The choice might improve in the future.)\n", - "WantButtons": [ - "Input Language" - ] + "Want": "\nCurrent settings:\n\nInput language: \"Hungarian\"\nInput language in ISO 639-3: \"hun\"\nTranslation languages in ISO 639-3: \"eng\",\"rus\",\"ukr\"\nTime Zone: UTC\n\nTo modify settings use one of the commands below:\n /language\n /timezone\n", + "WantButtons": null }, { - "Send": "b:Input Language", - "Want": "Enter input language of your choice. Supported are \"English\",\"Hungarian\"", + "Send": "/language", + "Want": "Enter input language of your choice. Supported are \"English\",\"German\",\"Hungarian\"", "WantButtons": null }, { "Send": "/stop", - "Want": "Exited settings", + "Want": "Stopped. Input the word to get it's definition.", + "WantButtons": null + }, + { + "Send": "/delete", + "Want": "Enter the word you want to delete from learning!", "WantButtons": null }, { - "Send": "/delete falu", + "Send": "falu", "Want": "Deleted \"falu\"!", "WantButtons": null }, @@ -192,21 +187,26 @@ }, { "Send": "/add", - "Want": "\nEnter the card you want to add in the format:\n\u003cfront of the card (word, expression, question)\u003e\n\u003cback of the card (can be multiline)\u003e\n", + "Want": "Enter front of the card (word, expression, question).", "WantButtons": null }, { "Send": "/stop", - "Want": "Cancelled addition", + "Want": "Stopped. Input the word to get it's definition.", "WantButtons": null }, { "Send": "/add", - "Want": "\nEnter the card you want to add in the format:\n\u003cfront of the card (word, expression, question)\u003e\n\u003cback of the card (can be multiline)\u003e\n", + "Want": "Enter front of the card (word, expression, question).", + "WantButtons": null + }, + { + "Send": "cardfront", + "Want": "Enter back of the card (definition, answer).", "WantButtons": null }, { - "Send": "cardfront\ncardback (definitions or what not)", + "Send": "cardback (definitions or what not)", "Want": "Added \"cardfront\" for learning!", "WantButtons": null }, @@ -225,4 +225,4 @@ "Don't know" ] } -] +] \ No newline at end of file From b411a70ce252164e9bef0dbd13985d9684ed9071 Mon Sep 17 00:00:00 2001 From: Attila Date: Mon, 26 Oct 2020 18:34:56 +0000 Subject: [PATCH 5/8] Implement script to populate database with usage examples. Using DB instead of loading content of csv files into the memory would reduce server's memory usage greatly. --- migrate/Dockerfile | 33 +++++ migrate/README.md | 19 +++ migrate/load.go | 333 +++++++++++++++++++++++++++++++++++++++++++ migrate/load_test.go | 79 ++++++++++ migrate/migrate.go | 67 --------- 5 files changed, 464 insertions(+), 67 deletions(-) create mode 100644 migrate/Dockerfile create mode 100644 migrate/README.md create mode 100644 migrate/load.go create mode 100644 migrate/load_test.go delete mode 100644 migrate/migrate.go diff --git a/migrate/Dockerfile b/migrate/Dockerfile new file mode 100644 index 0000000..ae772fa --- /dev/null +++ b/migrate/Dockerfile @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and then copy over the neede parts to create a small image +FROM golang:alpine AS builder + +RUN apk update && apk add --no-cache git gcc g++ ca-certificates apache2-utils +WORKDIR /go/src/words/migrate +COPY migrate/*.go ./ +RUN go get -d -v -tags netgo -installsuffix netgo +# netgo and ldflags makes sure that dns resolver and binary are statically +# linked giving the ability for smaller images. +RUN go build -tags netgo -installsuffix netgo -ldflags '-extldflags "-static"' -o /go/bin/migrate + +FROM scratch +# FIXME: Copying certificates looks a little bit hacky. Is there a better +# solution? +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /go/bin/migrate /go/bin/migrate +COPY data/*.csv data/ +# TODO: This should be moved to the CMD starting it up. +ENTRYPOINT ["/go/bin/migrate", "--db_path=/words-vol/db/db.sql", "--sentences=data/sentences.csv", "--links=data/links.csv"] diff --git a/migrate/README.md b/migrate/README.md new file mode 100644 index 0000000..878ab27 --- /dev/null +++ b/migrate/README.md @@ -0,0 +1,19 @@ +From the root of the project: + +```shell +sudo docker build -t loader . -f migrate/Dockerfile +sudo docker run --rm --name loader --mount source=words-vol,target=/words-vol/db/ loader +``` + +To save image: + +```shell +sudo docker save loader -o loader_image.tar +sudo chmod +r loader_image.tar +``` + + +Test image changes: +```shell +sudo docker run --rm --name temp_bash --mount source=words-vol,target=/words-vol/db/ -it ubuntu /bin/bash +``` diff --git a/migrate/load.go b/migrate/load.go new file mode 100644 index 0000000..3ed059e --- /dev/null +++ b/migrate/load.go @@ -0,0 +1,333 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// This is a script to load up usage examples into the database. +package main + +import ( + "bufio" + "database/sql" + "flag" + "fmt" + "log" + "os" + "strconv" + "strings" + "sync" + + _ "github.com/mattn/go-sqlite3" + "golang.org/x/sync/errgroup" +) + +type UsageFetcherOptions struct { + // Path to the file in csv with links between files, + LinksPath string + // Path to the file in csv with all the sentences. + // is an ISO 639-3 language code. + SentencesPath string +} + +type wordLang struct { + word string + lang string +} + +type sentence struct { + text string + lang string +} + +func (l *Loader) ReadAndLoad(opts UsageFetcherOptions) error { + sf, err := os.Open(opts.SentencesPath) + if err != nil { + return err + } + defer sf.Close() + + // Use single proc so that tx is single. Count and flush (commit + // transaction & create a new one) every 1M rows. + // proc would have sentence, word, translation methods. queries will be + // embedded. + p, err := newProc(l) + if err != nil { + return err + } + defer p.cleanup() + + c := make(chan string) + + processSentence := func() error { + for row := range c { + s := strings.Split(row, "\t") + if len(s) != 3 { + return fmt.Errorf("reading %q: wrond format for row %s", opts.SentencesPath, s) + } + id, err := strconv.ParseInt(s[0], 10, 64) + if err != nil { + return fmt.Errorf("reading %q: parsing id %q: %v", opts.SentencesPath, s[0], err) + } + lang, text := s[1], s[2] + if err := p.sentence(id, lang, text); err != nil { + return err + } + r := strings.NewReplacer( + ",", "", + ".", "", + "!", "", + ")", "", + "(", "", + "}", "", + "{", "", + "]", "", + "[", "", + ) + for _, w := range strings.Split(text, " ") { + word := strings.ToLower(r.Replace(w)) + if err := p.word(word, lang, id); err != nil { + return err + } + } + } + return nil + } + + scanner := bufio.NewScanner(sf) + go func() { + for scanner.Scan() { + c <- scanner.Text() + } + close(c) + }() + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading %q: %w", opts.SentencesPath, err) + } + eg := errgroup.Group{} + for n := 0; n < 16; n++ { + eg.Go(processSentence) + } + if err := eg.Wait(); err != nil { + return err + } + + lf, err := os.Open(opts.LinksPath) + if err != nil { + return err + } + defer lf.Close() + + scanner = bufio.NewScanner(lf) + for scanner.Scan() { + var ids []int64 + for _, i := range strings.Split(scanner.Text(), "\t") { + id, err := strconv.ParseInt(i, 10, 64) + if err != nil { + return fmt.Errorf("reading %q: parsing id %q: %v", opts.LinksPath, i, err) + } + ids = append(ids, id) + } + if len(ids) != 2 { + return fmt.Errorf("reading %q: wrond format for row %s", opts.LinksPath, scanner.Text()) + } + if err := p.translation(ids[0], ids[1]); err != nil { + return err + } + } + + return nil +} + +type Loader struct { + db *sql.DB + opts UsageFetcherOptions +} + +func NewLoader(dbPath, sPath, lPath string) (*Loader, error) { + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, fmt.Errorf("open: %v", err) + } + return &Loader{ + db, + UsageFetcherOptions{ + SentencesPath: sPath, + LinksPath: lPath, + }, + }, nil +} + +type proc struct { + db *sql.DB + stmt map[TableType]*sql.Stmt + + mu sync.Mutex + cnt map[TableType]int + processed int64 + tx *sql.Tx +} + +type TableType int + +const ( + SentencesTable TableType = iota + WordsTable + TranslationsTable +) + +func newProc(l *Loader) (p *proc, err error) { + p = new(proc) + p.tx, err = l.db.Begin() + if err != nil { + return + } + p.db = l.db + p.cnt = make(map[TableType]int) + p.stmt = make(map[TableType]*sql.Stmt) + for t, q := range map[TableType]string{ + SentencesTable: `INSERT OR REPLACE INTO Sentences(id, lang, text) + VALUES(?, ?, ?)`, + WordsTable: `INSERT OR REPLACE INTO Words(word, lang, sentence_id) + VALUES(?, ?, ?)`, + TranslationsTable: `INSERT OR REPLACE INTO Translations(id, translation_id) + VALUES(?, ?)`, + } { + p.stmt[t], err = l.db.Prepare(q) + if err != nil { + return + } + } + return +} + +func (p *proc) sentence(id int64, lang, text string) error { + err := p.row(SentencesTable, id, lang, text) + if err != nil { + err = fmt.Errorf("Row(%d, %s, %s): %v", id, lang, text, err) + } + return err +} +func (p *proc) word(word, lang string, sid int64) error { + err := p.row(WordsTable, word, lang, sid) + if err != nil { + err = fmt.Errorf("Row(%s, %s, %d): %v", word, lang, sid, err) + } + return err +} +func (p *proc) translation(id, tid int64) error { + err := p.row(TranslationsTable, id, tid) + if err != nil { + err = fmt.Errorf("Row(%d, %d): %v", id, tid, err) + } + return err +} + +func (p *proc) row(table TableType, args ...interface{}) error { + p.mu.Lock() + defer p.mu.Unlock() + p.cnt[table] += 1 + _, err := p.tx.Stmt(p.stmt[table]).Exec(args...) + if err != nil { + return err + } + p.processed += 1 + if p.processed%100_000 == 0 { + return p.commit() + } + return nil +} + +func (p *proc) commit() (err error) { + log.Printf("Flushing %d rows", p.processed) + if err := p.tx.Commit(); err != nil { + return err + } + log.Printf("In total wrote %v", p.cnt) + p.processed = 0 + p.tx, err = p.db.Begin() + return err +} + +func (p *proc) cleanup() { + if err := p.commit(); err != nil { + log.Printf("ERROR proc cleanup: %v", err) + } + for _, s := range p.stmt { + s.Close() + } + p.tx.Rollback() +} + +func (l *Loader) Load() error { + // word -> list of sentences (ids). OR word -> lang -> list of sentences. + // sentence id -> list of translation id. + // translation id -> sentence. + if _, err := l.db.Exec(` + PRAGMA foreign_keys = OFF; + + CREATE TABLE IF NOT EXISTS Sentences ( + id INTEGER PRIMARY KEY, + lang STRING, + text STRING + ); + + CREATE TABLE IF NOT EXISTS Translations ( + id INTEGER, + translation_id INTEGER, + FOREIGN KEY(id) REFERENCES Sentences(id), + FOREIGN KEY(translation_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS TranslationsIdIndex + ON Translations (id); + + CREATE TABLE IF NOT EXISTS Words ( + word STRING, + lang STRING, + sentence_id INTEGER, + FOREIGN KEY(sentence_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS WordLangIndex + ON Words (word, lang); + `); err != nil { + return err + } + + if err := l.ReadAndLoad(l.opts); err != nil { + return err + } + + //{ + // p, err := newProc(l, + // `INSERT OR REPLACE INTO Sentences(id, lang, text) + // VALUES(?, ?, ?)`) + // if err != nil { + // return err + // } + // defer p.cleanup() + return nil +} + +func main() { + db := flag.String("db_path", "../db.sql", "Path to the persistent sqlite3 database.") + sentences := flag.String("sentences", "../data/sentences.csv", "Path to the folder with sentences usage examples in csv format.") + links := flag.String("links", "../data/links.csv", "Path to the folder with links usage examples in csv format.") + flag.Parse() + + l, err := NewLoader(*db, *sentences, *links) + if err != nil { + log.Fatal(err) + } + if err := l.Load(); err != nil { + log.Fatal(err) + } +} diff --git a/migrate/load_test.go b/migrate/load_test.go new file mode 100644 index 0000000..31f67fe --- /dev/null +++ b/migrate/load_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "database/sql" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLoad(t *testing.T) { + dir, err := ioutil.TempDir("", "load_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + + count := func(table string) int32 { + t.Helper() + r := db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s;", table)) + var n int32 + if err := r.Scan(&n); err != nil { + t.Fatalf("count(%s): %v", table, err) + } + return n + } + + l, err := NewLoader(dbPath, "../testdata/sentences.csv", "../testdata/links.csv") + if err != nil { + t.Fatal(err) + } + if err := l.Load(); err != nil { + t.Fatal(err) + } + tables := []string{"Sentences", "Translations", "Words"} + got := make(map[string]int32) + for _, tb := range tables { + n := count(tb) + if n == 0 { + t.Errorf("%s: got %d want > 0", tb, n) + } + got[tb] = n + } + + // Concecutive calls to Load should result in no errors and be noops. + if err := l.Load(); err != nil { + t.Fatal(err) + } + want := got + for _, tb := range tables { + got[tb] = count(tb) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v want %v", got, want) + } + log.Printf("want: %v", want) +} diff --git a/migrate/migrate.go b/migrate/migrate.go deleted file mode 100644 index b92c428..0000000 --- a/migrate/migrate.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// This is a adhoc one-time run script to perform data migration on the -// database. -package main - -import ( - "database/sql" - "fmt" - "strings" - - _ "github.com/mattn/go-sqlite3" -) - -func Migrate(dbPath string) error { - db, err := sql.Open("sqlite3", dbPath) - if err != nil { - return fmt.Errorf("open: %v", err) - } - - rows, err := db.Query(` - SELECT word, definition - FROM Repetition;`) - defer rows.Close() - - nd := make(map[string]string) - for rows.Next() { - var w, d string - if err := rows.Scan(&w, &d); err != nil { - return fmt.Errorf("scan: %v", err) - } - nd[w] = strings.ReplaceAll(d, "********", w) - if !strings.HasPrefix(d, w) && !strings.HasPrefix(d, "*"+w) { - nd[w] = "*" + w + "*\n\n" + nd[w] - } - } - rows.Close() - - for w, d := range nd { - _, err := db.Exec(` - UPDATE Repetition - SET definition = $0 - WHERE word = $1;`, - d, w) - if err != nil { - return fmt.Errorf("update: %v", err) - } - } - return nil -} - -func main() { - fmt.Printf("Result of migration: %v\n", Migrate("../db.sql")) -} From 3138df8291e494add982d0cfc120a36260d52626 Mon Sep 17 00:00:00 2001 From: Attila Date: Mon, 26 Oct 2020 18:40:41 +0000 Subject: [PATCH 6/8] Use database instead of csv files for usage examples. Image size is reduced more than 10x which makes it easier to deploy new images faster. On the server with 2GB RAM Memory usage dropped from 73% to 13% after the new image was deployed. --- Dockerfile | 1 - commands.go | 13 +--- definer.go | 1 + e2e_test.go | 18 +++-- go.mod | 1 + go.sum | 2 + main.go | 6 +- testdata/e2e.json | 8 +- usage.go | 191 ++++++++++++++++------------------------------ usage_test.go | 79 +++++++++++++++++-- 10 files changed, 163 insertions(+), 157 deletions(-) diff --git a/Dockerfile b/Dockerfile index d368c2e..9cad5c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,5 @@ FROM scratch # solution? COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /go/bin/words /go/bin/words -COPY data/*.csv data/ # TODO: This should be moved to the CMD starting it up. ENTRYPOINT ["/go/bin/words", "--db_path=/words-vol/db/db.sql"] diff --git a/commands.go b/commands.go index 5f7e227..11877b8 100644 --- a/commands.go +++ b/commands.go @@ -123,11 +123,9 @@ type Commander struct { } type CommanderOptions struct { - useCache bool - dbPath string - stages []time.Duration - sentencesPath string - linksPath string // links between sentence and translations + useCache bool + dbPath string + stages []time.Duration } func escapeMarkdown(s string) string { @@ -168,10 +166,7 @@ func NewCommander(tm *Telegram, opts *CommanderOptions) (*Commander, error) { } // TODO: Can use errgroup if there is a need to paralelize. This is the // slowest step in initialization. - uf, err := NewUsageFetcher(UsageFetcherOptions{ - SentencesPath: opts.sentencesPath, - LinksPath: opts.linksPath, - }) + uf, err := NewUsageFetcher(opts.dbPath) if err != nil { return nil, fmt.Errorf("creating usage fetcher: %w", err) } diff --git a/definer.go b/definer.go index 456afe3..ec56aca 100644 --- a/definer.go +++ b/definer.go @@ -66,6 +66,7 @@ func (d *Definer) Define(word string, settings *Settings) (ds []string, err erro ex, err := d.usage.FetchExamples(word, settings.InputLanguageISO639_3, settings.TranslationLanguages) if err != nil { ex = nil + log.Printf("ERROR: FetchExamples(%s): %v", word, err) log.Printf("WARNING Did not find usage examples for %q", word) } msg := "*" + escapeMarkdown(word) + "*\n" diff --git a/e2e_test.go b/e2e_test.go index 1e5d083..56ca095 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "database/sql" "encoding/json" "fmt" "io/ioutil" @@ -131,17 +132,24 @@ cardfront t.Fatal(err) } defer os.RemoveAll(dir) - db := filepath.Join(dir, "tmpdb") + dbPath := filepath.Join(dir, "tmpdb") + + // Initiate db with some usage examples. + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(usageSQL); err != nil { + t.Fatal(err) + } fk := startFakeTelegram(t) defer fk.server.Close() tm := &Telegram{hc: *fk.server.Client()} c, err := NewCommander(tm, &CommanderOptions{ - useCache: true, - dbPath: db, - sentencesPath: "./testdata/sentences.csv", - linksPath: "./testdata/links.csv", + useCache: true, + dbPath: dbPath, stages: []time.Duration{ 0, 2 * time.Minute, diff --git a/go.mod b/go.mod index ac866e5..e2a27a8 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/google/go-cmp v0.4.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 ) diff --git a/go.sum b/go.sum index ba235cf..9045755 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnA golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 71a6caf..23258ff 100644 --- a/main.go +++ b/main.go @@ -43,10 +43,8 @@ func main() { log.Printf("db_path: %q", *db) ctx := context.Background() opts := &CommanderOptions{ - useCache: false, - dbPath: *db, - sentencesPath: "./data/sentences.csv", - linksPath: "./data/links.csv", + useCache: false, + dbPath: *db, stages: []time.Duration{ 20 * time.Second, 1 * time.Hour * 23, diff --git a/testdata/e2e.json b/testdata/e2e.json index ef7d702..2f5d368 100644 --- a/testdata/e2e.json +++ b/testdata/e2e.json @@ -21,14 +21,14 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Learn" ] }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Learn" ] @@ -40,7 +40,7 @@ }, { "Send": "fekete", - "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. Én vagyok a család fekete báránya\\.\n _I'm the black sheep of the family\\._\n\n2\\. A vörös ruhás nő figyelmen kívül hagyta a fekete ruhás férfit, és mobilon felhívta egy barátját\\.\n _The girl in the red dress ignored the man dressed in black and called a friend on her cellphone\\._\n\n3\\. Két kis nyuszi, egy fehér nyuszi meg egy fekete élt egy nagy erdőben\\.", + "Want": "*fekete*\n\n1\\. \\[*adjective*\\] black \\(absorbing all light and reflecting none\\)\n2\\. \\[*adjective*\\] black \\(pertaining to a dark\\-skinned ethnic group\\)\n3\\. \\[*adjective*\\] black \\(darker than other varieties, especially of fruits and drinks\\)\n4\\. \\[*adjective*\\] \\(figurative\\) tragic, mournful, black \\(causing great sadness or suffering\\)\n5\\. \\[*adjective*\\] \\(figurative\\) black \\(derived from evil forces, or performed with the intention of doing harm\\)\n6\\. \\[*adjective*\\] \\(figurative, in compounds\\) illegal \\(contrary to or forbidden by criminal law\\)\n7\\. \\[*noun*\\] black \\(color perceived in the absence of light\\)\n8\\. \\[*noun*\\] black clothes \\(especially as mourning attire\\)\n_\\[truncated 3 definitions\\]_\n\nUsage examples:\n\n1\\. fekete kutya\n _black dog_\n\n2\\. fekete kutya\n _чорний собака_\n\n3\\. fekete disznó", "WantButtons": [ "Reset progress" ] @@ -89,7 +89,7 @@ }, { "Send": "falu", - "Want": "*falu*\n\n1\\. \\[*noun*\\] village\nA világ egy falu\\.The world is a village\\.\nSynonym: község\nHypernyms: település, helység\nHyponyms: törpefalu \\(\u003c100\\), aprófalu \\(100–500\\), kisfalu \\(500–1,000\\), középfalu \\(1,000–2,000\\), nagyfalu \\(2,000–5,000\\), óriásfalu \\(5,000–10,000 of population\\)\nCoordinate term: város \\(town or city\\)\n\nUsage examples:\n\n1\\. Minden falu egy személyt delegálhatott\\.\n\n2\\. Mindegyik falu szép volt\\.\n\n3\\. A gátépítés előtt egy falu volt itt\\.", + "Want": "*falu*\n\n1\\. \\[*noun*\\] village\nA világ egy falu\\.The world is a village\\.\nSynonym: község\nHypernyms: település, helység\nHyponyms: törpefalu \\(\u003c100\\), aprófalu \\(100–500\\), kisfalu \\(500–1,000\\), középfalu \\(1,000–2,000\\), nagyfalu \\(2,000–5,000\\), óriásfalu \\(5,000–10,000 of population\\)\nCoordinate term: város \\(town or city\\)\n\nDidn't find usage examples\\.", "WantButtons": [ "Learn" ] diff --git a/usage.go b/usage.go index 6912b27..09c86db 100644 --- a/usage.go +++ b/usage.go @@ -14,117 +14,31 @@ package main import ( - "bufio" + "database/sql" "fmt" - "log" - "os" - "sort" - "strconv" "strings" ) -type UsageFetcherOptions struct { - // Path to the file in csv with links between files, - LinksPath string - // Path to the file in csv with all the sentences. - // is an ISO 639-3 language code. - SentencesPath string -} - // Usage is struct that is able to extract usage examples from the tatoeba // datasets. type UsageFetcher struct { - // word -> sentence IDs - ws map[string][]int64 - // id -> sentence - ss map[int64]*sentence - // id -> translation ids - ts map[int64][]int64 + db *sql.DB } type sentence struct { - text string - lang string - translationIDs []int64 + text string + lang string } -func NewUsageFetcher(opts UsageFetcherOptions) (*UsageFetcher, error) { - // FIXME: Check that Language is not in TranslationLanguages - sf, err := os.Open(opts.SentencesPath) +// NewUsageFetcher creates a new usage fetcher. +func NewUsageFetcher(dbPath string) (*UsageFetcher, error) { + db, err := sql.Open("sqlite3", dbPath) if err != nil { return nil, err } - defer sf.Close() - - whitelist := make(map[string]bool) - for _, v := range SupportedInputLanguages { - whitelist[v.InputLanguageISO639_3] = true - for k, v := range v.TranslationLanguages { - if v { - whitelist[k] = true - } - } - } - - ss := make(map[int64]*sentence) - ws := make(map[string][]int64) - scanner := bufio.NewScanner(sf) - for scanner.Scan() { - s := strings.Split(scanner.Text(), "\t") - if len(s) != 3 { - return nil, fmt.Errorf("reading %q: wrond format for row %s", opts.SentencesPath, s) - } - id, err := strconv.ParseInt(s[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("reading %q: parsing id %q: %v", opts.SentencesPath, s[0], err) - } - // Skip unsupported languages and translations for optimization. Might - // need to be removed if this thing is made more general. - if !whitelist[s[1]] { - continue - } - ss[id] = &sentence{ - lang: s[1], - text: s[2], - } - // FIXME: Words will not be correctly extracted if there are - // punctuations. - for _, w := range strings.Split(s[2], " ") { - ws[w] = append(ws[w], id) - } - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading %q: %w", opts.SentencesPath, err) - } - - lf, err := os.Open(opts.LinksPath) - if err != nil { - return nil, err - } - defer lf.Close() - - ts := make(map[int64][]int64) - scanner = bufio.NewScanner(lf) - for scanner.Scan() { - var ids []int64 - for _, i := range strings.Split(scanner.Text(), "\t") { - id, err := strconv.ParseInt(i, 10, 64) - if err != nil { - return nil, fmt.Errorf("reading %q: parsing id %q: %v", opts.LinksPath, i, err) - } - ids = append(ids, id) - } - if len(ids) != 2 { - return nil, fmt.Errorf("reading %q: wrond format for row %s", opts.LinksPath, scanner.Text()) - } - ts[ids[0]] = append(ts[ids[0]], ids[1]) - } - log.Printf("Number of translations: %d", len(ts)) - + // Schema for the db can be found in migrate/load.go return &UsageFetcher{ - ws: ws, - ss: ss, - ts: ts, + db: db, }, nil } @@ -136,45 +50,68 @@ type UsageExample struct { // FIXME: Too many parameters // language is a langugage of the word in ISO 639-3 format. func (u *UsageFetcher) FetchExamples(word, language string, translationLanguages map[string]bool) ([]*UsageExample, error) { - // TODO: rank examples by complexity and extract the simplest ones: - // 1) for each word calculate it's complexity by the number of sentences it's - // used in (more sentences -> simpler words) - // 2) the sentence is simpler if it contains simpler words. Maybe average - // word simplicity to not disqualify long sentences. - // - // TODO: Prioritize using sentences with the most translations. + var tls []interface{} + for k, v := range translationLanguages { + if v { + tls = append(tls, k) + } + } + // We use Sprintf only to insert variable number of ?, so it cannot cause + // SQL injection. + q := fmt.Sprintf(` + SELECT DISTINCT s.text, ts.text + FROM + Words + INNER JOIN + Sentences s ON Words.sentence_id = s.id + LEFT JOIN + Translations ON Words.sentence_id = Translations.id + LEFT JOIN + Sentences ts ON Translations.translation_id = ts.id + WHERE + Words.word = ? + AND s.lang = ? + AND (ts.lang IS NULL OR ts.lang IN (?%s)) + -- If possible get definitions with translations first. + ORDER BY CASE WHEN ts.text IS NULL THEN 1 ELSE 0 END + LIMIT 3;`, strings.Repeat(", ?", len(tls)-1)) + args := append([]interface{}{ + word, language, + }, tls...) + rows, err := u.db.Query(q, args...) + if err == sql.ErrNoRows { + return nil, fmt.Errorf("no sentences match %s", word) + } + if err != nil { + return nil, err + } + defer rows.Close() + var ex []*UsageExample - for _, i := range u.ws[word] { - s := u.ss[i] - if s.lang != language { - continue + for rows.Next() { + var ( + e string + t sql.NullString + ) + if err := rows.Scan(&e, &t); err != nil { + return nil, err } var tr []string - for _, i := range u.ts[i] { - s, ok := u.ss[i] - if !ok { - log.Printf("ERROR: inconsistent sentences, no id %d found in sentences", i) - continue - } - if _, ok := translationLanguages[s.lang]; !ok { - continue - } - tr = append(tr, s.text) - } - if len(tr) > 4 { - tr = tr[:4] + if t.Valid { + tr = append(tr, t.String) } ex = append(ex, &UsageExample{ - Text: s.text, + Text: e, Translations: tr, }) } - if len(ex) == 0 { - return nil, fmt.Errorf("no sentences match %s", word) - } - sort.Slice(ex, func(i, j int) bool { return len(ex[i].Translations) > len(ex[j].Translations) }) - if len(ex) > 3 { - ex = ex[:3] - } + + // TODO: rank examples by complexity and extract the simplest ones: + // 1) for each word calculate it's complexity by the number of sentences it's + // used in (more sentences -> simpler words) + // 2) the sentence is simpler if it contains simpler words. Maybe average + // word simplicity to not disqualify long sentences. + // + // TODO: Prioritize using sentences with the most translations. return ex, nil } diff --git a/usage_test.go b/usage_test.go index dfc1588..497d528 100644 --- a/usage_test.go +++ b/usage_test.go @@ -14,20 +14,85 @@ package main import ( + "io/ioutil" + "os" + "path/filepath" "strings" "testing" ) +const usageSQL = ` + PRAGMA foreign_keys = OFF; + + CREATE TABLE IF NOT EXISTS Sentences ( + id INTEGER PRIMARY KEY, + lang STRING, + text STRING + ); + + CREATE TABLE IF NOT EXISTS Translations ( + id INTEGER, + translation_id INTEGER, + FOREIGN KEY(id) REFERENCES Sentences(id), + FOREIGN KEY(translation_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS TranslationsIdIndex + ON Translations (id); + + CREATE TABLE IF NOT EXISTS Words ( + word STRING, + lang STRING, + sentence_id INTEGER, + FOREIGN KEY(sentence_id) REFERENCES Sentences(id) + ); + CREATE INDEX IF NOT EXISTS WordLangIndex + ON Words (word, lang); + + INSERT OR REPLACE INTO Sentences(id, lang, text) VALUES + (1, "hun", "fekete kutya"), + (2, "hun", "fekete disznó"), + (3, "hun", "fekete macska fehér asztalon"), + (4, "hun", "fehér disznó"), + (5, "hun", "fehér fal"), + (6, "hun", "fehér haj"), + (7, "eng", "black dog"), + (8, "eng", "white pig"), + (9, "ukr", "чорний собака"); + INSERT OR REPLACE INTO Words(word, lang, sentence_id) VALUES + ("fekete", "hun", 1), + ("fekete", "hun", 2), + ("fekete", "hun", 3), + ("fehér", "hun", 3), + ("fehér", "hun", 4), + ("fehér", "hun", 5), + ("fehér", "hun", 6); + INSERT OR REPLACE INTO Translations(id, translation_id) VALUES + (1, 7), + (1, 9), + (7, 1), + (9, 1), + (4, 8), + (8, 4); + ` + func TestUsageFetcher(t *testing.T) { - uf, err := NewUsageFetcher(UsageFetcherOptions{ - SentencesPath: "testdata/sentences.csv", - LinksPath: "testdata/links.csv", - }) + dir, err := ioutil.TempDir("", "repetition") + if err != nil { + t.Fatal(err) + } + t.Logf("Temp dir: %q", dir) + defer os.RemoveAll(dir) + + dbPath := filepath.Join(dir, "tmpdb") + uf, err := NewUsageFetcher(dbPath) if err != nil { t.Fatal(err) } + if _, err := uf.db.Exec(usageSQL); err != nil { + t.Fatal(err) + } - for _, word := range []string{"fekete", "fehér", "közös"} { + for _, word := range []string{"fekete", "fehér"} { t.Run(word, func(t *testing.T) { ex, err := uf.FetchExamples(word, "hun", map[string]bool{ "eng": true, @@ -38,8 +103,8 @@ func TestUsageFetcher(t *testing.T) { t.Fatal(err) } // check that each returned example contains asked query. - if len(ex) == 0 { - t.Fatalf("len(usage examples): got %d; want > 0", len(ex)) + if len(ex) != 3 { + t.Fatalf("len(usage examples): got %d; want 3", len(ex)) } for _, e := range ex { if !strings.Contains(e.Text, word) { From 7c8ad897b7d63a382e94bdba3a4d139b0005bef8 Mon Sep 17 00:00:00 2001 From: Attila Date: Mon, 26 Oct 2020 18:41:10 +0000 Subject: [PATCH 7/8] Add .dockerignore Makes building images slightly faster. --- .dockerignore | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e336e8b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +*.swp +*.swo +db.sql +words +migrate/migrate +words_image.tar +loader_image.tar +data/*.bz2 +commands.md +cover.out +testdata/* +backups/* +.vscode +.git +.cache From 1c32c2f56e5d2f010bde3b8b5cca5f84519bc412 Mon Sep 17 00:00:00 2001 From: Attila Date: Mon, 26 Oct 2020 18:41:41 +0000 Subject: [PATCH 8/8] Extend .gitignore with loader images This is mostly for convenience. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index bd589bd..5381cad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ secret.go *.swo db.sql words +migrate/migrate words_image.tar +loader_image.tar data/*.csv data/*.bz2 commands.md