From 324de3d99f22bbce918e0246daa6caf90a0fc734 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 10:10:46 +1200 Subject: [PATCH 1/3] Include mailbox name in error message --- storage/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/utils.go b/storage/utils.go index 7be037630..73725da0d 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -76,7 +76,7 @@ func pruneCron() { if err := db.Delete(clover.NewQuery(m). Sort(clover.SortOption{Field: "Created", Direction: 1}). Limit(limit)); err != nil { - logger.Log().Warnf("Error pruning: %s", err.Error()) + logger.Log().Warnf("Error pruning %s: %s", m, err.Error()) continue } elapsed := time.Since(start) From 335b0f3876e66da7bb06e6c39afb016068da1ac9 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 19:57:44 +1200 Subject: [PATCH 2/3] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96e9dc14e..f8505911b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Mailpit is inspired by [MailHog](#why-rewrite-mailhog), but much, much faster. - Runs completely on a single binary - SMTP server (default `0.0.0.0:1025`) - Web UI to view emails (HTML format, text, source and MIME attachments, default `0.0.0.0:8025`) -- Real-time web UI updates using websockets for new mail +- Real-time web UI updates using web sockets for new mail - Email storage in either memory or disk (using [CloverDB](https://github.com/ostafen/clover)) - note that in-memory has a physical limit of 1MB per email - Configurable automatic email pruning (default keeps the most recent 500 emails) - Fast SMTP processing & storing - approximately 300-600 emails per second depending on CPU, network speed & email size From a85a74bb9a03dafeaaaf3e2211cb796f195773e3 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 19:58:31 +1200 Subject: [PATCH 3/3] Feature: Unread statistics --- data/mailbox.go | 6 +++ server/api.go | 33 +++++--------- server/ui-src/App.vue | 12 +++-- storage/database.go | 49 ++++++++++---------- storage/stats.go | 103 ++++++++++++++++++++++++++++++++++++++++++ storage/utils.go | 1 + 6 files changed, 156 insertions(+), 48 deletions(-) create mode 100644 storage/stats.go diff --git a/data/mailbox.go b/data/mailbox.go index ae58ac290..d8f269fd9 100644 --- a/data/mailbox.go +++ b/data/mailbox.go @@ -16,3 +16,9 @@ type WebsocketNotification struct { Type string Data interface{} } + +// MailboxStats struct for quick mailbox total/read lookups +type MailboxStats struct { + Total int + Unread int +} diff --git a/server/api.go b/server/api.go index e39083b0b..c00f850e1 100644 --- a/server/api.go +++ b/server/api.go @@ -12,10 +12,11 @@ import ( ) type messagesResult struct { - Total int `json:"total"` - Count int `json:"count"` - Start int `json:"start"` - Items []data.Summary `json:"items"` + Total int `json:"total"` + Unread int `json:"unread"` + Count int `json:"count"` + Start int `json:"start"` + Items []data.Summary `json:"items"` } // Return a list of available mailboxes @@ -49,18 +50,15 @@ func apiListMailbox(w http.ResponseWriter, r *http.Request) { return } - total, err := storage.Count(mailbox) - if err != nil { - httpError(w, err.Error()) - return - } + stats := storage.StatsGet(mailbox) var res messagesResult res.Start = start res.Items = messages res.Count = len(res.Items) - res.Total = total + res.Total = stats.Total + res.Unread = stats.Unread bytes, _ := json.Marshal(res) w.Header().Add("Content-Type", "application/json") @@ -92,24 +90,15 @@ func apiSearchMailbox(w http.ResponseWriter, r *http.Request) { return } - total, err := storage.Count(mailbox) - if err != nil { - httpError(w, err.Error()) - return - } - - // total := limit - // count := len(messages) - // if total > count { - // total = count - // } + stats := storage.StatsGet(mailbox) var res messagesResult res.Start = start res.Items = messages res.Count = len(messages) - res.Total = total + res.Total = stats.Total + res.Unread = stats.Unread bytes, _ := json.Marshal(res) w.Header().Add("Content-Type", "application/json") diff --git a/server/ui-src/App.vue b/server/ui-src/App.vue index 25b4dccd2..7de51a1e4 100644 --- a/server/ui-src/App.vue +++ b/server/ui-src/App.vue @@ -15,6 +15,7 @@ export default { items: [], limit: 50, total: 0, + unread: 0, start: 0, search: "", searching: false, @@ -71,6 +72,7 @@ export default { self.get(uri, params, function(response){ self.total = response.data.total; + self.unread = response.data.unread; self.count = response.data.count; self.start = response.data.start; self.items = response.data.items; @@ -119,7 +121,10 @@ export default { self.get(uri, params, function(response) { for (let i in self.items) { if (self.items[i].ID == self.currentPath) { - self.items[i].Read = true; + if (!self.items[i].Read) { + self.items[i].Read = true; + self.unread--; + } } } let d = response.data; @@ -208,6 +213,7 @@ export default { } } self.total++; + self.unread++; } else if (response.Type == "prune") { // messages have been deleted, reload messages to adjust self.scrollInPlace = true; @@ -323,8 +329,8 @@ export default { Inbox - - {{ formatNumber(total) }} + + {{ formatNumber(unread) }} diff --git a/storage/database.go b/storage/database.go index d3fdfda2c..78845b25e 100644 --- a/storage/database.go +++ b/storage/database.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "regexp" + "strings" "syscall" "time" @@ -99,24 +100,20 @@ func ListMailboxes() ([]data.MailboxSummary, error) { results := []data.MailboxSummary{} for _, m := range mailboxes { - - total, err := Count(m) - if err != nil { - return nil, err + // ignore *_data collections + if strings.HasSuffix(m, "_data") { + continue } - unread, err := CountUnread(m) - if err != nil { - return nil, err - } + stats := StatsGet(m) mb := data.MailboxSummary{} mb.Name = m mb.Slug = m - mb.Total = total - mb.Unread = unread + mb.Total = stats.Total + mb.Unread = stats.Unread - if total > 0 { + if mb.Total > 0 { q, err := db.FindFirst( clover.NewQuery(m).Sort(clover.SortOption{Field: "Created", Direction: -1}), ) @@ -172,7 +169,7 @@ func CreateMailbox(name string) error { } } - return nil + return statsRefresh(name) } // Store will store a message in the database and return the unique ID @@ -223,6 +220,8 @@ func Store(mailbox string, b []byte) (string, error) { return "", err } + statsAddNewMessage(mailbox) + count++ if count%100 == 0 { logger.Log().Infof("%d messages added (%s per 100)", count, time.Since(per100start)) @@ -441,11 +440,16 @@ func GetMessage(mailbox, id string) (*data.Message, error) { obj.HTML = html - updates := make(map[string]interface{}) - updates["Read"] = true + msg, err := db.FindById(mailbox, id) + if err == nil && !msg.Get("Read").(bool) { + updates := make(map[string]interface{}) + updates["Read"] = true - if err := db.UpdateById(mailbox, id, updates); err != nil { - return nil, err + if err := db.UpdateById(mailbox, id, updates); err != nil { + return nil, err + } + + statsReadOneMessage(mailbox) } return &obj, nil @@ -507,6 +511,8 @@ func UnreadMessage(mailbox, id string) error { updates := make(map[string]interface{}) updates["Read"] = false + statsUnreadOneMessage(mailbox) + return db.UpdateById(mailbox, id, updates) } @@ -516,6 +522,8 @@ func DeleteOneMessage(mailbox, id string) error { return err } + statsDeleteOneMessage(mailbox) + return db.DeleteById(mailbox+"_data", id) } @@ -545,13 +553,8 @@ func DeleteAllMessages(mailbox string) error { } } - // if err := db.Delete(clover.NewQuery(mailbox)); err != nil { - // return err - // } - - // if err := db.Delete(clover.NewQuery(mailbox + "_data")); err != nil { - // return err - // } + // resets stats for mailbox + statsRefresh(mailbox) elapsed := time.Since(totalStart) logger.Log().Infof("Deleted %d messages from %s in %s", totalMessages, mailbox, elapsed) diff --git a/storage/stats.go b/storage/stats.go new file mode 100644 index 000000000..4c3fc5774 --- /dev/null +++ b/storage/stats.go @@ -0,0 +1,103 @@ +package storage + +import ( + "sync" + + "github.com/axllent/mailpit/data" + "github.com/axllent/mailpit/logger" + "github.com/ostafen/clover/v2" +) + +var ( + mailboxStats = map[string]data.MailboxStats{} + statsLock = sync.RWMutex{} +) + +// StatsGet returns the total/unread statistics for a mailbox +func StatsGet(mailbox string) data.MailboxStats { + statsLock.Lock() + defer statsLock.Unlock() + s, ok := mailboxStats[mailbox] + if !ok { + return data.MailboxStats{ + Total: 0, + Unread: 0, + } + } + return s +} + +// Refresh will completely refresh the existing stats for a given mailbox +func statsRefresh(mailbox string) error { + logger.Log().Debugf("[stats] refreshing stats for %s", mailbox) + + total, err := db.Count(clover.NewQuery(mailbox)) + if err != nil { + return err + } + + unread, err := db.Count(clover.NewQuery(mailbox).Where(clover.Field("Read").IsFalse())) + if err != nil { + return nil + } + + statsLock.Lock() + mailboxStats[mailbox] = data.MailboxStats{ + Total: total, + Unread: unread, + } + statsLock.Unlock() + + return nil +} + +func statsAddNewMessage(mailbox string) { + statsLock.Lock() + s, ok := mailboxStats[mailbox] + if ok { + mailboxStats[mailbox] = data.MailboxStats{ + Total: s.Total + 1, + Unread: s.Unread + 1, + } + } + statsLock.Unlock() +} + +// Deleting one will always mean it was read +func statsDeleteOneMessage(mailbox string) { + statsLock.Lock() + s, ok := mailboxStats[mailbox] + if ok { + mailboxStats[mailbox] = data.MailboxStats{ + Total: s.Total - 1, + Unread: s.Unread, + } + } + statsLock.Unlock() +} + +// Mark one message as read +func statsReadOneMessage(mailbox string) { + statsLock.Lock() + s, ok := mailboxStats[mailbox] + if ok { + mailboxStats[mailbox] = data.MailboxStats{ + Total: s.Total, + Unread: s.Unread - 1, + } + } + statsLock.Unlock() +} + +// Mark one message as unread +func statsUnreadOneMessage(mailbox string) { + statsLock.Lock() + s, ok := mailboxStats[mailbox] + if ok { + mailboxStats[mailbox] = data.MailboxStats{ + Total: s.Total, + Unread: s.Unread + 1, + } + } + statsLock.Unlock() +} diff --git a/storage/utils.go b/storage/utils.go index 73725da0d..8cfca725b 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -81,6 +81,7 @@ func pruneCron() { } elapsed := time.Since(start) logger.Log().Infof("Pruned %d messages from %s in %s", limit, m, elapsed) + statsRefresh(m) if !strings.HasSuffix(m, "_data") { websockets.Broadcast("prune", nil) }