From ce64f50933bdb21073a6ee04b3687b5bee1e2f8c Mon Sep 17 00:00:00 2001 From: Yufan Sheng Date: Thu, 28 Dec 2023 16:38:29 +0800 Subject: [PATCH] fix: change the progress store when enable the filter logic. --- internal/fetcher/fetcher.go | 49 ++++++++++++++++++++++++++++++++---- internal/fetcher/service.go | 15 ----------- internal/fetcher/telegram.go | 4 +++ 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index db995f6..9ef8d93 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -5,7 +5,10 @@ import ( "fmt" "os" "path/filepath" + "strconv" + "strings" "sync" + "time" "github.com/bookstairs/bookhunter/internal/driver" "github.com/bookstairs/bookhunter/internal/file" @@ -49,7 +52,12 @@ func (f *fetcher) Download() error { // Create download progress with rate limit. if f.processFile == "" { - f.processFile = defaultProgressFile + if len(f.Keywords) == 0 { + f.processFile = defaultProgressFile + } else { + // Avoid the download progress overloading. + f.processFile = strconv.FormatInt(time.Now().Unix(), 10) + defaultProgressFile + } } rate := f.RateLimit * f.Thread f.progress, err = progress.NewProgress(f.InitialBookID, size, rate, filepath.Join(configPath, f.processFile)) @@ -92,7 +100,7 @@ func (f *fetcher) Download() error { } // startDownload will start a download thread. -func (f *fetcher) startDownload() { +func (f *fetcher) startDownload() { //nolint:gocyclo thread: for { bookID := f.progress.AcquireBookID() @@ -102,7 +110,7 @@ thread: break thread } - // Start download the given book ID. + // Start downloading the given book ID. // The error will be sent to the channel. // Acquire the available file formats @@ -119,10 +127,20 @@ thread: log.Warnf("[%d/%d] No downloadable files found.", bookID, f.progress.Size()) } + // Filter the name, skip the progress if the name isn't the desired one. + if len(formats) != 0 && len(f.Keywords) != 0 { + formats = f.filterNames(formats) + if len(formats) == 0 { + log.Warnf("[%d/%d] The files found by the given keywords", bookID, f.progress.Size()) + // No need to save the download progress. + continue + } + } + // Download the file by formats one by one. for format, share := range formats { err := f.downloadFile(bookID, format, share) - for retry := 0; err != nil && retry < f.Retry; retry++ { + for retry := 0; err != nil && !errors.Is(err, ErrFileNotExist) && retry < f.Retry; retry++ { fmt.Printf("Download book id %d failed: %v, retry (%d/%d)\n", bookID, err, retry, f.Retry) err = f.downloadFile(bookID, format, share) } @@ -165,7 +183,7 @@ func (f *fetcher) filterFormats(formats map[file.Format]driver.Share) map[file.F fs := make(map[file.Format]driver.Share) for format, share := range formats { for _, vf := range f.Formats { - if format == vf && matchKeywords(share.FileName, f.Keywords) { + if format == vf { fs[format] = share break } @@ -173,3 +191,24 @@ func (f *fetcher) filterFormats(formats map[file.Format]driver.Share) map[file.F } return fs } + +func (f *fetcher) filterNames(formats map[file.Format]driver.Share) map[file.Format]driver.Share { + fs := make(map[file.Format]driver.Share) + for format, share := range formats { + if matchKeywords(share.FileName, f.Keywords) { + fs[format] = share + } + } + return fs +} + +func matchKeywords(title string, keywords []string) bool { + for _, keyword := range keywords { + // Should we support the regular expression? + if strings.Contains(title, keyword) { + return true + } + } + + return false +} diff --git a/internal/fetcher/service.go b/internal/fetcher/service.go index 7f1e252..ce9b7da 100644 --- a/internal/fetcher/service.go +++ b/internal/fetcher/service.go @@ -2,7 +2,6 @@ package fetcher import ( "fmt" - "strings" "github.com/bookstairs/bookhunter/internal/driver" "github.com/bookstairs/bookhunter/internal/file" @@ -20,20 +19,6 @@ type service interface { fetch(int64, file.Format, driver.Share, file.Writer) error } -func matchKeywords(title string, keywords []string) bool { - if len(keywords) == 0 { - return true - } - - for _, keyword := range keywords { - if strings.Contains(title, keyword) { - return true - } - } - - return false -} - // newService is the endpoint for creating all the supported download service. func newService(c *Config) (service, error) { switch c.Category { diff --git a/internal/fetcher/telegram.go b/internal/fetcher/telegram.go index a18b715..ca5d31b 100644 --- a/internal/fetcher/telegram.go +++ b/internal/fetcher/telegram.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/gotd/td/tg" @@ -31,6 +32,9 @@ func newTelegramService(config *Config) (service, error) { // Change the process file name. config.processFile = strings.ReplaceAll(channelID, "/", "_") + ".db" + if len(config.Keywords) == 0 { + config.processFile = strconv.FormatInt(time.Now().Unix(), 10) + config.processFile + } tel, err := telegram.New(channelID, mobile, appID, appHash, sessionPath, config.Proxy) if err != nil {