From b37a24fdcf505a5b34f3ef75341c7d0f14418544 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 25 Aug 2024 00:01:17 +1200 Subject: [PATCH 1/7] Code cleanup --- cmd/ingest.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index cab2419e9..f35e45981 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "fmt" "io" "net/mail" "net/smtp" @@ -13,8 +14,6 @@ import ( "github.com/axllent/mailpit/internal/logger" sendmail "github.com/axllent/mailpit/sendmail/cmd" "github.com/spf13/cobra" - "golang.org/x/text/language" - "golang.org/x/text/message" ) var ( @@ -36,7 +35,6 @@ The --recent flag will only consider files with a modification date within the l var count int var total int var per100start = time.Now() - p := message.NewPrinter(language.English) for _, a := range args { err := filepath.Walk(a, @@ -117,8 +115,7 @@ The --recent flag will only consider files with a modification date within the l count++ total++ if count%100 == 0 { - formatted := p.Sprintf("%d", total) - logger.Log().Infof("[%s] 100 messages in %s", formatted, time.Since(per100start)) + logger.Log().Infof("[%s] 100 messages in %s", format(total), time.Since(per100start)) per100start = time.Now() } @@ -149,3 +146,29 @@ func isFile(path string) bool { return true } + +// Format a an integer 10000 => 10,000 +func format(n int) string { + in := fmt.Sprintf("%d", n) + numOfDigits := len(in) + if n < 0 { + numOfDigits-- // First character is the - sign (not a digit) + } + numOfCommas := (numOfDigits - 1) / 3 + + out := make([]byte, len(in)+numOfCommas) + if n < 0 { + in, out[0] = in[1:], '-' + } + + for i, j, k := len(in)-1, len(out)-1, 0; ; i, j = i-1, j-1 { + out[j] = in[i] + if i == 0 { + return string(out) + } + if k++; k == 3 { + j, k = j-1, 0 + out[j] = ',' + } + } +} From 259d71122bd454b6c08d9182daa2c9315eb2d9f2 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 1 Sep 2024 08:56:45 +1200 Subject: [PATCH 2/7] Chore: Do not recenter selected messages in sidebar on every new message --- server/ui-src/views/MessageView.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/server/ui-src/views/MessageView.vue b/server/ui-src/views/MessageView.vue index 288735fea..df7c6e4d4 100644 --- a/server/ui-src/views/MessageView.vue +++ b/server/ui-src/views/MessageView.vue @@ -224,7 +224,6 @@ export default { this.liveLoaded++ this.messagesList.unshift(data) - this.scrollSidebarToCurrent() }, // handler for websocket message updates From be1d2bcb2841a62f9ee47731980ed196a14593c4 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 1 Sep 2024 18:35:42 +1200 Subject: [PATCH 3/7] Fix: Disable automatic HTML/Text character detection when charset is provided (#348) --- go.mod | 2 +- go.sum | 4 ++-- internal/storage/messages.go | 12 +++++++++--- internal/storage/reindex.go | 4 +++- server/apiv1/api.go | 4 +++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 03922b837..e310e2ce6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 - github.com/jhillyerd/enmime v1.2.0 + github.com/jhillyerd/enmime v1.3.0 github.com/klauspost/compress v1.17.9 github.com/kovidgoyal/imaging v1.6.3 github.com/leporo/sqlf v1.4.0 diff --git a/go.sum b/go.sum index 755a05f05..03127a28d 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= -github.com/jhillyerd/enmime v1.2.0 h1:dIu1IPEymQgoT2dzuB//ttA/xcV40NMPpQtmd4wslHk= -github.com/jhillyerd/enmime v1.2.0/go.mod h1:FRFuUPCLh8PByQv+8xRcLO9QHqaqTqreYhopv5eyk4I= +github.com/jhillyerd/enmime v1.3.0 h1:LV5kzfLidiOr8qRGIpYYmUZCnhrPbcFAnAFUnWn99rw= +github.com/jhillyerd/enmime v1.3.0/go.mod h1:6c6jg5HdRRV2FtvVL69LjiX1M8oE0xDX9VEhV3oy4gs= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kovidgoyal/imaging v1.6.3 h1:iNPpv7ygiaB/NOztc6APMT7yr9UwBS+rOZwIbAdtyY8= diff --git a/internal/storage/messages.go b/internal/storage/messages.go index e73b8f12c..199e91823 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -27,8 +27,10 @@ import ( // Store will save an email to the database tables. // Returns the database ID of the saved message. func Store(body *[]byte) (string, error) { + parser := enmime.NewParser(enmime.DisableCharacterDetection(true)) + // Parse message body with enmime - env, err := enmime.ReadEnvelope(bytes.NewReader(*body)) + env, err := parser.ReadEnvelope(bytes.NewReader(*body)) if err != nil { logger.Log().Warnf("[message] %s", err.Error()) return "", nil @@ -245,7 +247,9 @@ func GetMessage(id string) (*Message, error) { r := bytes.NewReader(raw) - env, err := enmime.ReadEnvelope(r) + parser := enmime.NewParser(enmime.DisableCharacterDetection(true)) + + env, err := parser.ReadEnvelope(r) if err != nil { return nil, err } @@ -396,7 +400,9 @@ func GetAttachmentPart(id, partID string) (*enmime.Part, error) { r := bytes.NewReader(raw) - env, err := enmime.ReadEnvelope(r) + parser := enmime.NewParser(enmime.DisableCharacterDetection(true)) + + env, err := parser.ReadEnvelope(r) if err != nil { return nil, err } diff --git a/internal/storage/reindex.go b/internal/storage/reindex.go index 8d040d281..3b7d873bb 100644 --- a/internal/storage/reindex.go +++ b/internal/storage/reindex.go @@ -49,6 +49,8 @@ func ReindexAll() { Metadata string } + parser := enmime.NewParser(enmime.DisableCharacterDetection(true)) + for _, ids := range chunks { updates := []updateStruct{} @@ -61,7 +63,7 @@ func ReindexAll() { r := bytes.NewReader(raw) - env, err := enmime.ReadEnvelope(r) + env, err := parser.ReadEnvelope(r) if err != nil { logger.Log().Errorf("[message] %s", err.Error()) continue diff --git a/server/apiv1/api.go b/server/apiv1/api.go index 14821f924..ee086161e 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -559,7 +559,9 @@ func HTMLCheck(w http.ResponseWriter, r *http.Request) { e := bytes.NewReader(raw) - msg, err := enmime.ReadEnvelope(e) + parser := enmime.NewParser(enmime.DisableCharacterDetection(true)) + + msg, err := parser.ReadEnvelope(e) if err != nil { httpError(w, err.Error()) return From 32185e3abe33c2cedd841f6f5c51c25f28d66102 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 1 Sep 2024 19:18:33 +1200 Subject: [PATCH 4/7] Chore: Update Go dependencies --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index e310e2ce6..425df90da 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( golang.org/x/sys v0.24.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a // indirect - modernc.org/libc v1.59.9 // indirect + modernc.org/libc v1.60.1 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect modernc.org/strutil v1.2.0 // indirect diff --git a/go.sum b/go.sum index 03127a28d..72807800f 100644 --- a/go.sum +++ b/go.sum @@ -197,16 +197,16 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.20.7 h1:skrinQsjxWfvj6nbC3ztZPJy+NuwmB3hV9zX/pthNYQ= -modernc.org/ccgo/v4 v4.20.7/go.mod h1:UOkI3JSG2zT4E2ioHlncSOZsXbuDCZLvPi3uMlZT5GY= +modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4= +modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M= modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a h1:CfbpOLEo2IwNzJdMvE8aiRbPMxoTpgAJeyePh0SmO8M= modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.59.9 h1:k+nNDDakwipimgmJ1D9H466LhFeSkaPPycAs1OZiDmY= -modernc.org/libc v1.59.9/go.mod h1:EY/egGEU7Ju66eU6SBqCNYaFUDuc4npICkMWnU5EE3A= +modernc.org/libc v1.60.1 h1:at373l8IFRTkJIkAU85BIuUoBM4T1b51ds0E1ovPG2s= +modernc.org/libc v1.60.1/go.mod h1:xJuobKuNxKH3RUatS7GjR+suWj+5c2K7bi4m/S5arOY= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= From 48195b004efd3ea8273683087a27f8dade17340f Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 1 Sep 2024 19:51:07 +1200 Subject: [PATCH 5/7] Chore: Update node dependencies --- package-lock.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index bd68a8ca3..ad63e80d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1112,9 +1112,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", - "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -1234,9 +1234,9 @@ ] }, "node_modules/bootstrap5-tags": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/bootstrap5-tags/-/bootstrap5-tags-1.7.4.tgz", - "integrity": "sha512-eQWu6/EHGf45R8EFdTCk8JQeY5oqfvQvEaYNZmFttnN56YOFpD7U+S5snEngsxZArwj57BLa9Yf0/QOggXJhcg==" + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/bootstrap5-tags/-/bootstrap5-tags-1.7.5.tgz", + "integrity": "sha512-EqCpdjD/UdZVYdlgcWfQKU68x7AtUs3lSnl9/u1xW2kVc193nqE4QOyJ5fAvc4i4t365LerRG87kptMpsD1PlQ==" }, "node_modules/brace-expansion": { "version": "2.0.1", @@ -1382,9 +1382,9 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" }, "node_modules/dayjs": { - "version": "1.11.12", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.12.tgz", - "integrity": "sha512-Rt2g+nTbLlDWZTwwrIXjy9MeiZmSDI375FvZs72ngxx8PDC6YXOeR3q5LAuPzjZQxhiWdRKac7RKV+YyQYfYIg==" + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", + "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==" }, "node_modules/decompress-response": { "version": "6.0.0", From 1c8f0bf136d52f5c26e00dfe1f06afa5976e231a Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 1 Sep 2024 19:52:44 +1200 Subject: [PATCH 6/7] Chore: Update caniemail database --- internal/htmlcheck/caniemail-data.json | 236 ++++++++++++++++++++++++- 1 file changed, 230 insertions(+), 6 deletions(-) diff --git a/internal/htmlcheck/caniemail-data.json b/internal/htmlcheck/caniemail-data.json index 2f33b6921..a236a8c01 100644 --- a/internal/htmlcheck/caniemail-data.json +++ b/internal/htmlcheck/caniemail-data.json @@ -1,6 +1,6 @@ { "api_version":"1.0.4", - "last_update_date":"2024-07-29 15:27:49 +0000", + "last_update_date":"2024-08-31 16:00:28 +0000", "nicenames":{"family":{"gmail":"Gmail","outlook":"Outlook","yahoo":"Yahoo! Mail","apple-mail":"Apple Mail","aol":"AOL","thunderbird":"Mozilla Thunderbird","microsoft":"Microsoft","samsung-email":"Samsung Email","sfr":"SFR","orange":"Orange","protonmail":"ProtonMail","hey":"HEY","mail-ru":"Mail.ru","fastmail":"Fastmail","laposte":"LaPoste.net","t-online-de":"T-online.de","free-fr":"Free.fr","gmx":"GMX","web-de":"WEB.DE","ionos-1and1":"1&1","rainloop":"RainLoop","wp-pl":"WP.pl"},"platform":{"desktop-app":"Desktop","desktop-webmail":"Desktop Webmail","mobile-webmail":"Mobile Webmail","webmail":"Webmail","ios":"iOS","android":"Android","windows":"Windows","macos":"macOS","windows-mail":"Windows Mail","outlook-com":"Outlook.com"},"support":{"supported":"Supported","mitigated":"Partially supported","unsupported":"Not supported","unknown":"Support unknown","mixed":"Mixed support"},"category":{"html":"HTML","css":"CSS","image":"Image formats","others":"Others"}}, "data":[ { @@ -691,6 +691,22 @@ "notes_by_num":null }, + { + "slug":"css-comments", + "title":"CSS comments", + "description":"Adds explanatory notes to the code or to prevent the browser from interpreting specific parts of the style sheet", + "url":"https://www.caniemail.com/features/css-comments/", + "category":"css", + "tags":[], + "keywords":null, + "last_test_date":"2024-04-25", + "test_url":"https://www.caniemail.com/tests/css-comments.html", + "test_results_url":"https://testi.at/proj/n4ayign05k6cozot6", + "stats":{"apple-mail":{"macos":{"2024-04":"y"},"ios":{"2024-04":"y"}},"gmail":{"desktop-webmail":{"2024-04":"y"},"ios":{"2024-04":"y"},"android":{"2024-04":"a #2"},"mobile-webmail":{"2024-04":"a #4"}},"orange":{"desktop-webmail":{"2024-08":"n #6"},"ios":{"2024-08":"n #6"},"android":{"2024-08":"n #6"}},"outlook":{"windows":{"2013":"y","2016":"y","2019":"y","2021":"y"},"windows-mail":{"2024-04":"y"},"macos":{"2024-04":"y"},"outlook-com":{"2024-04":"y","2024-01":"y"},"ios":{"2024-04":"y"},"android":{"2024-04":"y"}},"yahoo":{"desktop-webmail":{"2024-04":"y"},"ios":{"2024-04":"a #1"},"android":{"2024-04":"a #1"}},"aol":{"desktop-webmail":{"2024-04":"y"},"ios":{"2024-04":"a #1"},"android":{"2024-04":"a #1"}},"samsung-email":{"android":{"2024-04":"y"}},"sfr":{"desktop-webmail":{"2024-08":"a #5"},"ios":{"2024-08":"a #5"},"android":{"2024-08":"a #5"}},"thunderbird":{"macos":{"2024-04":"y"}},"protonmail":{"desktop-webmail":{"2024-04":"u"},"ios":{"2024-04":"u"},"android":{"2024-04":"u"}},"hey":{"desktop-webmail":{"2024-04":"u"}},"mail-ru":{"desktop-webmail":{"2024-04":"a #3"}},"fastmail":{"desktop-webmail":{"2024-04":"u"}},"laposte":{"desktop-webmail":{"2024-08":"a #5"}},"gmx":{"desktop-webmail":{"2024-04":"n"},"ios":{"2024-04":"u"},"android":{"2024-04":"u"}},"web-de":{"desktop-webmail":{"2024-04":"n"},"ios":{"2024-04":"u"},"android":{"2024-04":"u"}},"ionos-1and1":{"desktop-webmail":{"2024-04":"u"},"android":{"2024-04":"u"}}}, + "notes":null, + "notes_by_num":{"1":"Buggy. The first in the HTML is removed, so comment needs to be in the `