Merge branch 'main' into validate_username

.github/workflows/build_binaries.yml

@@ -9,13 +9,13 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Set up Golang
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
           check-latest: true
-      - name: Checkout
-        uses: actions/checkout@v4
       - name: Compile binaries
         env:
           CGO_ENABLED: 0

.github/workflows/codeql-analysis.yml

@@ -22,8 +22,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        language: [ 'go', 'javascript' ]
 
     steps:
     - name: Checkout repository

.github/workflows/linters.yml

@@ -29,7 +29,6 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
-      - run: "go vet ./..."
       - uses: golangci/golangci-lint-action@v6
         with:
           args: >

.github/workflows/tests.yml

@@ -17,14 +17,18 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
         go-version: ["1.23.x"]
     steps:
+    - name: Checkout
+      uses: actions/checkout@v4
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Run unit tests
+    - name: Run unit tests with coverage and race conditions checking
+      if: matrix.os == 'ubuntu-latest'
       run: make test
+    - name: Run unit tests without coverage and race conditions checking
+      if: matrix.os != 'ubuntu-latest'
+      run: go test ./...
 
   integration-tests:
     name: Integration Tests
@@ -40,12 +44,12 @@ jobs:
         - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
+    - name: Checkout
+      uses: actions/checkout@v4
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
         go-version: "1.23.x"
-    - name: Checkout
-      uses: actions/checkout@v4
     - name: Install Postgres client
       run: sudo apt update && sudo apt install -y postgresql-client
     - name: Run integration tests

ChangeLog

@@ -1,3 +1,64 @@
+Version 2.2.4 (December 20, 2024)
+---------------------------------
+
+* test(rewrite): add unit test for referer rewrite function
+* refactor(subscription): use `strings.HasSuffix` instead of a regex in `FindSubscriptionsFromYouTubePlaylistPage`
+* refactor(sanitizer): use `token.String()` instead of `html.EscapeString(token.Data)`
+* refactor(sanitizer): simplify `isValidTag`
+* refactor(sanitizer): simplify `hasRequiredAttributes`
+* refactor(sanitizer): remove condition because `config.Opts` is guaranteed to never be nil
+* refactor(sanitizer): remove a now-useless function after refactoring
+* refactor(sanitizer): refactor conditions to highlight their similitude, enabling further refactoring
+* refactor(sanitizer): optimize `strip_tags.go`
+* refactor(sanitizer): micro-optimizations of `srcset.go`
+* refactor(sanitizer): merge two conditions
+* refactor(sanitizer): inline a function in `sanitizeAttributes` and fix a bug in it
+* refactor(sanitizer): inline a condition in `sanitizeSrcsetAttr`
+* refactor(sanitizer): improve `rewriteIframeURL()`
+* refactor(sanitizer): Google+ isn't a thing anymore
+* refactor(sanitizer): change the scope of a variable
+* refactor(rewriter): replace regex with URL parsing for referrer override
+* refactor(rewriter): avoid the use of regex in `addDynamicImage`
+* refactor(rewrite): remove unused function arguments
+* refactor(readability): various improvements and optimizations
+* refactor(readability): simplify the regexes in `readability.go`
+* refactor(processor): use URL parsing instead of a regex
+* refactor(processor): improve the `rewrite` URL rule regex
+* refactor(locale): delay parsing of translations until they're used
+* refactor(js): factorise a line in `app.js`
+* refactor(handler): delay `store.UserByID()` as much as possible
+* refactor(css): replace `-ms-text-size-adjust` with `text-size-adjust`
+* refactor(css): remove `-webkit-clip-path`
+* refactor(css): factorise `.pagination-next` and `.pagination-last` together
+* refactor: use a better construct than `doc.Find(…).First()`
+* refactor: use `min/max` instead of `math.Min/math.Max`
+* refactor: refactor `internal/reader/readability/testdata`
+* refactor: optimize `sanitizeAttributes`
+* refactor: get rid of `numberOfPluralFormsPerLanguage` test-only variable
+* fix(storage): replace timezone function call with view
+* fix(consistency): align feed modification behavior between API and UI
+* fix(ci): fix grammar in pull-request template
+* fix: load icon from site URL instead of feed URL
+* fix: feed icon from xml ignored during force refresh
+* feat(rewrite)!: remove `parse_markdown` rewrite rule
+* feat(mediaproxy): update predefined referer spoofing rules for restricted media resources
+* feat(locale): update translations to clarify readeck URL instead of readeck API endpoint
+* feat(locale): update German translations
+* feat(locale): update Chinese translations
+* feat(apprise): update `SendNotification` to handle multiple entries and add logging
+* feat(apprise): add title in notification request body
+* feat: resize favicons before storing them in the database
+* feat: optionally fetch watch time from YouTube API instead of website
+* feat: only show the commit URL if it's not empty on `/about`
+* feat: add predefined scraper rules for `arstechnica.com`
+* feat: add date-based entry filtering rules
+* chore: remove `blog.laravel.com` rewrite rule
+* build(deps): bump `library/alpine` in `/packaging/docker/alpine` to `3.21`
+* build(deps): bump `golang.org/x/term` from `0.26.0` to `0.27.0`
+* build(deps): bump `golang.org/x/net` from `0.31.0` to `0.33.0`
+* build(deps): bump `golang.org/x/crypto` from `0.30.0` to `0.31.0`
+* build(deps): bump `github.com/tdewolff/minify/v2` from `2.21.1` to `2.21.2`
+
 Version 2.2.3 (November 10, 2024)
 ---------------------------------
 

go.mod

@@ -14,10 +14,9 @@ require (
 	github.com/tdewolff/minify/v2 v2.21.2
 	golang.org/x/crypto v0.31.0
 	golang.org/x/image v0.23.0
-	golang.org/x/net v0.32.0
+	golang.org/x/net v0.33.0
 	golang.org/x/oauth2 v0.24.0
 	golang.org/x/term v0.27.0
-	golang.org/x/text v0.21.0
 )
 
 require (
@@ -42,6 +41,7 @@ require (
 	github.com/tdewolff/parse/v2 v2.7.19 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 )
 

go.sum

@@ -79,8 +79,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
-golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
 golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

internal/reader/icon/finder.go

@@ -206,7 +206,7 @@ func resizeIcon(icon *model.Icon) *model.Icon {
 		slog.Warn("unable to decode the metadata of the icon", slog.Any("error", err))
 		return icon
 	}
-	if config.Height <= 16 && config.Width <= 16 {
+	if config.Height <= 32 && config.Width <= 32 {
 		slog.Debug("icon don't need to be rescaled", slog.Int("height", config.Height), slog.Int("width", config.Width))
 		return icon
 	}
@@ -227,7 +227,7 @@ func resizeIcon(icon *model.Icon) *model.Icon {
 		return icon
 	}
 
-	dst := image.NewRGBA(image.Rect(0, 0, 16, 16))
+	dst := image.NewRGBA(image.Rect(0, 0, 32, 32))
 	draw.BiLinear.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
 
 	var b bytes.Buffer

internal/reader/icon/finder_test.go

@@ -146,7 +146,7 @@ func TestResizeIconSmallGif(t *testing.T) {
 }
 
 func TestResizeIconPng(t *testing.T) {
-	data, err := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAHElEQVR42mP8z/C/noFCwDhqyKgho4aMGkIlQwBrHSpf28Yx+gAAAABJRU5ErkJggg==")
+	data, err := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAACEAAAAhCAYAAABX5MJvAAAALUlEQVR42u3OMQEAAAgDoJnc6BpjDyRgcrcpGwkJCQkJCQkJCQkJCQkJCYmyB7NfUj/Kk4FkAAAAAElFTkSuQmCC")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -157,15 +157,15 @@ func TestResizeIconPng(t *testing.T) {
 	resizedIcon := resizeIcon(&icon)
 
 	if bytes.Equal(data, resizedIcon.Content) {
-		t.Fatalf("Didn't convert png of 17x17")
+		t.Fatalf("Didn't convert png of 33x33")
 	}
 
 	config, _, err := image.DecodeConfig(bytes.NewReader(resizedIcon.Content))
 	if err != nil {
 		t.Fatalf("Couln't decode resulting png: %v", err)
 	}
 
-	if config.Height != 16 || config.Width != 16 {
+	if config.Height != 32 || config.Width != 32 {
 		t.Fatalf("Was expecting an image of 16x16, got %dx%d", config.Width, config.Height)
 	}
 }

internal/reader/processor/processor.go

@@ -10,6 +10,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/tdewolff/minify/v2"
+	"github.com/tdewolff/minify/v2/html"
+
 	"miniflux.app/v2/internal/config"
 	"miniflux.app/v2/internal/metric"
 	"miniflux.app/v2/internal/model"
@@ -20,9 +23,6 @@ import (
 	"miniflux.app/v2/internal/reader/scraper"
 	"miniflux.app/v2/internal/reader/urlcleaner"
 	"miniflux.app/v2/internal/storage"
-
-	"github.com/tdewolff/minify/v2"
-	"github.com/tdewolff/minify/v2/html"
 )
 
 var customReplaceRuleRegex = regexp.MustCompile(`rewrite\("([^"]+)"\|"([^"]+)"\)`)
@@ -141,6 +141,9 @@ func isBlockedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool
 
 			var match bool
 			switch parts[0] {
+			case "EntryDate":
+				datePattern := parts[1]
+				match = isDateMatchingPattern(entry.Date, datePattern)
 			case "EntryTitle":
 				match, _ = regexp.MatchString(parts[1], entry.Title)
 			case "EntryURL":
@@ -211,6 +214,9 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry, user *model.User) bool
 
 			var match bool
 			switch parts[0] {
+			case "EntryDate":
+				datePattern := parts[1]
+				match = isDateMatchingPattern(entry.Date, datePattern)
 			case "EntryTitle":
 				match, _ = regexp.MatchString(parts[1], entry.Title)
 			case "EntryURL":
@@ -462,3 +468,44 @@ func minifyEntryContent(entryContent string) string {
 
 	return entryContent
 }
+
+func isDateMatchingPattern(entryDate time.Time, pattern string) bool {
+	if pattern == "future" {
+		return entryDate.After(time.Now())
+	}
+
+	parts := strings.SplitN(pattern, ":", 2)
+	if len(parts) != 2 {
+		return false
+	}
+
+	operator := parts[0]
+	dateStr := parts[1]
+
+	switch operator {
+	case "before":
+		targetDate, err := time.Parse("2006-01-02", dateStr)
+		if err != nil {
+			return false
+		}
+		return entryDate.Before(targetDate)
+	case "after":
+		targetDate, err := time.Parse("2006-01-02", dateStr)
+		if err != nil {
+			return false
+		}
+		return entryDate.After(targetDate)
+	case "between":
+		dates := strings.Split(dateStr, ",")
+		if len(dates) != 2 {
+			return false
+		}
+		startDate, err1 := time.Parse("2006-01-02", dates[0])
+		endDate, err2 := time.Parse("2006-01-02", dates[1])
+		if err1 != nil || err2 != nil {
+			return false
+		}
+		return entryDate.After(startDate) && entryDate.Before(endDate)
+	}
+	return false
+}

internal/reader/processor/processor_test.go

@@ -75,6 +75,12 @@ func TestAllowEntries(t *testing.T) {
 		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Example", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, true},
 		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)example"}, true},
 		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Author: "Different", Tags: []string{"example", "something else"}}, &model.User{KeepFilterEntryRules: "EntryAuthor=(?i)example\nEntryTag=(?i)Test"}, false},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, true},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Now().Add(-24 * time.Hour)}, &model.User{KeepFilterEntryRules: "EntryDate=future"}, false},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 14, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=before:2024-03-15"}, true},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 16, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=after:2024-03-15"}, true},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, true},
+		{&model.Feed{ID: 1, BlocklistRules: ""}, &model.Entry{Date: time.Date(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, &model.User{KeepFilterEntryRules: "EntryDate=between:2024-03-01,2024-03-15"}, false},
 	}
 
 	for _, tc := range scenarios {

internal/reader/rewrite/rewrite_functions.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
+	"unicode"
 
 	"miniflux.app/v2/internal/config"
 
@@ -26,6 +27,24 @@ var (
 	textLinkRegex  = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`)
 )
 
+// titlelize returns a copy of the string s with all Unicode letters that begin words
+// mapped to their Unicode title case.
+func titlelize(s string) string {
+	// A closure is used here to remember the previous character
+	// so that we can check if there is a space preceding the current
+	// character.
+	previous := ' '
+	return strings.Map(
+		func(current rune) rune {
+			if unicode.IsSpace(previous) {
+				previous = current
+				return unicode.ToTitle(current)
+			}
+			previous = current
+			return current
+		}, strings.ToLower(s))
+}
+
 func addImageTitle(entryContent string) string {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
 	if err != nil {

internal/reader/rewrite/rewriter.go

@@ -11,9 +11,6 @@ import (
 
 	"miniflux.app/v2/internal/model"
 	"miniflux.app/v2/internal/urllib"
-
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 )
 
 type rule struct {
@@ -94,7 +91,7 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
 	case "remove_tables":
 		entry.Content = removeTables(entry.Content)
 	case "remove_clickbait":
-		entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title))
+		entry.Title = titlelize(entry.Title)
 	}
 }
 

internal/reader/rewrite/rules.go

@@ -50,6 +50,8 @@ func GetRefererForURL(u string) string {
 	}
 
 	switch parsedUrl.Hostname() {
+	case "moyu.im":
+		return "https://i.jandan.net"
 	case "i.pximg.net":
 		return "https://www.pixiv.net"
 	case "sp1.piokok.com":
@@ -60,13 +62,19 @@ func GetRefererForURL(u string) string {
 		return "https://weibo.com"
 	case "img.hellogithub.com":
 		return "https://hellogithub.com"
+	case "bjp.org.cn":
+		return "https://bjp.org.cn"
+	case "appinn.com":
+		return "https://appinn.com"
 	}
 
 	switch {
 	case strings.HasSuffix(parsedUrl.Hostname(), ".sinaimg.cn"):
 		return "https://weibo.com"
 	case strings.HasSuffix(parsedUrl.Hostname(), ".cdninstagram.com"):
 		return "https://www.instagram.com"
+	case strings.HasSuffix(parsedUrl.Hostname(), ".moyu.im"):
+		return "https://i.jandan.net"
 	}
 
 	return ""

internal/validator/user.go

@@ -241,7 +241,7 @@ func validateMediaPlaybackRate(mediaPlaybackRate float64) *locale.LocalizedError
 
 func isValidFilterRules(filterEntryRules string, filterType string) *locale.LocalizedError {
 	// Valid Format: FieldName=RegEx\nFieldName=RegEx...
-	fieldNames := []string{"EntryTitle", "EntryURL", "EntryCommentsURL", "EntryContent", "EntryAuthor", "EntryTag"}
+	fieldNames := []string{"EntryTitle", "EntryURL", "EntryCommentsURL", "EntryContent", "EntryAuthor", "EntryTag", "EntryDate"}
 
 	rules := strings.Split(filterEntryRules, "\n")
 	for i, rule := range rules {