From d9925a3cf694fb3d8eb0ac93b10c71efb813a0ab Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 14:52:20 +1000 Subject: [PATCH 1/9] Adding ability to specify the maximum download concurrency for the download missing episodes functionality. Defaults to `5` (the current hardcoded value). --- client/settings.html | 6 ++++++ controllers/pages.go | 1 + db/podcast.go | 1 + service/podcastService.go | 5 +++-- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/client/settings.html b/client/settings.html index 48665a3..c727bbf 100644 --- a/client/settings.html +++ b/client/settings.html @@ -104,6 +104,10 @@

Settings

Base URL (if accessing Podgrab using a URL. Without trailing /. Leave empty if not using or unsure.) + @@ -181,6 +185,7 @@

More Info

downloadEpisodeImages:self.downloadEpisodeImages, generateNFOFile:self.generateNFOFile, dontDownloadDeletedFromDisk:self.dontDownloadDeletedFromDisk, + maxDownloadConcurrency:self.maxDownloadConcurrency, baseUrl:self.baseUrl, }) .then(function(response){ @@ -225,6 +230,7 @@

More Info

downloadEpisodeImages:{{.setting.DownloadEpisodeImages }}, generateNFOFile:{{ .setting.GenerateNFOFile }}, dontDownloadDeletedFromDisk:{{ .setting.DontDownloadDeletedFromDisk }}, + maxDownloadConcurrency:{{ .setting.maxDownloadConcurrency }}, baseUrl: {{ .setting.BaseUrl }}, }, diff --git a/controllers/pages.go b/controllers/pages.go index 2034e22..9b805d5 100644 --- a/controllers/pages.go +++ b/controllers/pages.go @@ -31,6 +31,7 @@ type SettingModel struct { GenerateNFOFile bool `form:"generateNFOFile" json:"generateNFOFile" query:"generateNFOFile"` DontDownloadDeletedFromDisk bool `form:"dontDownloadDeletedFromDisk" json:"dontDownloadDeletedFromDisk" query:"dontDownloadDeletedFromDisk"` BaseUrl string `form:"baseUrl" json:"baseUrl" query:"baseUrl"` + MaxDownloadConcurrency bool `form:"maxDownloadConcurrency" json:"maxDownloadConcurrency" query:"maxDownloadConcurrency"` } var searchOptions = map[string]string{ diff --git a/db/podcast.go b/db/podcast.go index 0962dd1..6614087 100644 --- a/db/podcast.go +++ b/db/podcast.go @@ -87,6 +87,7 @@ type Setting struct { GenerateNFOFile bool `gorm:"default:false"` DontDownloadDeletedFromDisk bool `gorm:"default:false"` BaseUrl string + MaxDownloadConcurrency int `gorm:"default:5"` } type Migration struct { Base diff --git a/service/podcastService.go b/service/podcastService.go index 345d2a2..5dbd3d3 100644 --- a/service/podcastService.go +++ b/service/podcastService.go @@ -537,7 +537,7 @@ func DownloadMissingEpisodes() error { SetPodcastItemAsDownloaded(item.ID, url) }(item, *setting) - if index%5 == 0 { + if index%setting.MaxDownloadConcurrency == 0 { wg.Wait() } } @@ -764,7 +764,7 @@ func GetSearchFromPodcastIndex(pod *podcastindex.Podcast) *model.CommonSearchRes func UpdateSettings(downloadOnAdd bool, initialDownloadCount int, autoDownload bool, appendDateToFileName bool, appendEpisodeNumberToFileName bool, darkMode bool, downloadEpisodeImages bool, - generateNFOFile bool, dontDownloadDeletedFromDisk bool, baseUrl string) error { + generateNFOFile bool, dontDownloadDeletedFromDisk bool, baseUrl string, maxDownloadConcurrency int) error { setting := db.GetOrCreateSetting() setting.AutoDownload = autoDownload @@ -777,6 +777,7 @@ func UpdateSettings(downloadOnAdd bool, initialDownloadCount int, autoDownload b setting.GenerateNFOFile = generateNFOFile setting.DontDownloadDeletedFromDisk = dontDownloadDeletedFromDisk setting.BaseUrl = baseUrl + setting.MaxDownloadConcurrency = maxDownloadConcurrency return db.UpdateSettings(setting) } From a000d19d029d7ac4f16832837d3987c0fdda9543 Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 14:57:44 +1000 Subject: [PATCH 2/9] Forgot to pass argument in single place --- controllers/podcast.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controllers/podcast.go b/controllers/podcast.go index de692e8..cb114ef 100644 --- a/controllers/podcast.go +++ b/controllers/podcast.go @@ -628,7 +628,9 @@ func UpdateSetting(c *gin.Context) { err = service.UpdateSettings(model.DownloadOnAdd, model.InitialDownloadCount, model.AutoDownload, model.AppendDateToFileName, model.AppendEpisodeNumberToFileName, - model.DarkMode, model.DownloadEpisodeImages, model.GenerateNFOFile, model.DontDownloadDeletedFromDisk, model.BaseUrl) + model.DarkMode, model.DownloadEpisodeImages, model.GenerateNFOFile, model.DontDownloadDeletedFromDisk, model.BaseUrl, + model.MaxDownloadConcurrency + ) if err == nil { c.JSON(200, gin.H{"message": "Success"}) From 822ec69f12f9d6ef1d9ec9e79f705738a95a77b3 Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 15:01:12 +1000 Subject: [PATCH 3/9] Fixing type of setting and formatting --- controllers/pages.go | 2 +- controllers/podcast.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/pages.go b/controllers/pages.go index 9b805d5..32feaa2 100644 --- a/controllers/pages.go +++ b/controllers/pages.go @@ -31,7 +31,7 @@ type SettingModel struct { GenerateNFOFile bool `form:"generateNFOFile" json:"generateNFOFile" query:"generateNFOFile"` DontDownloadDeletedFromDisk bool `form:"dontDownloadDeletedFromDisk" json:"dontDownloadDeletedFromDisk" query:"dontDownloadDeletedFromDisk"` BaseUrl string `form:"baseUrl" json:"baseUrl" query:"baseUrl"` - MaxDownloadConcurrency bool `form:"maxDownloadConcurrency" json:"maxDownloadConcurrency" query:"maxDownloadConcurrency"` + MaxDownloadConcurrency int `form:"maxDownloadConcurrency" json:"maxDownloadConcurrency" query:"maxDownloadConcurrency"` } var searchOptions = map[string]string{ diff --git a/controllers/podcast.go b/controllers/podcast.go index cb114ef..71da974 100644 --- a/controllers/podcast.go +++ b/controllers/podcast.go @@ -629,7 +629,7 @@ func UpdateSetting(c *gin.Context) { err = service.UpdateSettings(model.DownloadOnAdd, model.InitialDownloadCount, model.AutoDownload, model.AppendDateToFileName, model.AppendEpisodeNumberToFileName, model.DarkMode, model.DownloadEpisodeImages, model.GenerateNFOFile, model.DontDownloadDeletedFromDisk, model.BaseUrl, - model.MaxDownloadConcurrency + model.MaxDownloadConcurrency, ) if err == nil { c.JSON(200, gin.H{"message": "Success"}) From 2128152b6fc988f0b922c1e0b4d6c07721cd7508 Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 15:14:39 +1000 Subject: [PATCH 4/9] Fixing up settings binding --- client/settings.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/settings.html b/client/settings.html index c727bbf..472f9a4 100644 --- a/client/settings.html +++ b/client/settings.html @@ -104,9 +104,9 @@

Settings

Base URL (if accessing Podgrab using a URL. Without trailing /. Leave empty if not using or unsure.) - - + @@ -187,6 +190,7 @@

More Info

dontDownloadDeletedFromDisk:self.dontDownloadDeletedFromDisk, baseUrl:self.baseUrl, maxDownloadConcurrency:self.maxDownloadConcurrency, + userAgent:self.userAgent, }) .then(function(response){ Vue.toasted.show('Settings saved successfully.' ,{ @@ -232,6 +236,7 @@

More Info

dontDownloadDeletedFromDisk:{{ .setting.DontDownloadDeletedFromDisk }}, baseUrl: {{ .setting.BaseUrl }}, maxDownloadConcurrency:{{ .setting.MaxDownloadConcurrency }}, + userAgent:{{ .setting.UserAgent}}, }, }) diff --git a/controllers/pages.go b/controllers/pages.go index 32feaa2..2b13de3 100644 --- a/controllers/pages.go +++ b/controllers/pages.go @@ -32,6 +32,7 @@ type SettingModel struct { DontDownloadDeletedFromDisk bool `form:"dontDownloadDeletedFromDisk" json:"dontDownloadDeletedFromDisk" query:"dontDownloadDeletedFromDisk"` BaseUrl string `form:"baseUrl" json:"baseUrl" query:"baseUrl"` MaxDownloadConcurrency int `form:"maxDownloadConcurrency" json:"maxDownloadConcurrency" query:"maxDownloadConcurrency"` + UserAgent string `form:"userAgent" json:"userAgent" query:"userAgent"` } var searchOptions = map[string]string{ diff --git a/controllers/podcast.go b/controllers/podcast.go index 71da974..199f4a3 100644 --- a/controllers/podcast.go +++ b/controllers/podcast.go @@ -629,7 +629,7 @@ func UpdateSetting(c *gin.Context) { err = service.UpdateSettings(model.DownloadOnAdd, model.InitialDownloadCount, model.AutoDownload, model.AppendDateToFileName, model.AppendEpisodeNumberToFileName, model.DarkMode, model.DownloadEpisodeImages, model.GenerateNFOFile, model.DontDownloadDeletedFromDisk, model.BaseUrl, - model.MaxDownloadConcurrency, + model.MaxDownloadConcurrency, model.UserAgent, ) if err == nil { c.JSON(200, gin.H{"message": "Success"}) diff --git a/db/podcast.go b/db/podcast.go index 6614087..8a33823 100644 --- a/db/podcast.go +++ b/db/podcast.go @@ -88,6 +88,7 @@ type Setting struct { DontDownloadDeletedFromDisk bool `gorm:"default:false"` BaseUrl string MaxDownloadConcurrency int `gorm:"default:5"` + UserAgent string } type Migration struct { Base diff --git a/service/fileService.go b/service/fileService.go index 4c9fc56..e2ca745 100644 --- a/service/fileService.go +++ b/service/fileService.go @@ -27,7 +27,14 @@ func Download(link string, episodeTitle string, podcastName string, prefix strin return "", errors.New("Download path empty") } client := httpClient() - resp, err := client.Get(link) + + req, err := http.NewRequest("GET", link, nil) + req.Header.Add("User-Agent", "") + if err != nil { + Logger.Errorw("Error creating request: "+link, err) + } + + resp, err := client.Do(req) if err != nil { Logger.Errorw("Error getting response: "+link, err) return "", err @@ -102,7 +109,13 @@ func DownloadPodcastCoverImage(link string, podcastName string) (string, error) return "", errors.New("Download path empty") } client := httpClient() - resp, err := client.Get(link) + req, err := getRequest(link) + if err != nil { + Logger.Errorw("Error creating request: "+link, err) + return "", err + } + + resp, err := client.Do(req) if err != nil { Logger.Errorw("Error getting response: "+link, err) return "", err @@ -139,7 +152,13 @@ func DownloadImage(link string, episodeId string, podcastName string) (string, e return "", errors.New("Download path empty") } client := httpClient() - resp, err := client.Get(link) + req, err := getRequest(link) + if err != nil { + Logger.Errorw("Error creating request: "+link, err) + return "", err + } + + resp, err := client.Do(req) if err != nil { Logger.Errorw("Error getting response: "+link, err) return "", err @@ -326,6 +345,20 @@ func httpClient() *http.Client { return &client } +func getRequest(url string) (*http.Request, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + setting := db.GetOrCreateSetting() + if len(setting.UserAgent) > 0 { + req.Header.Add("User-Agent", setting.UserAgent) + } + + return req, nil +} + func createFolder(folder string, parent string) string { folder = cleanFileName(folder) //str := stringy.New(folder) diff --git a/service/podcastService.go b/service/podcastService.go index 5dbd3d3..ac91a5e 100644 --- a/service/podcastService.go +++ b/service/podcastService.go @@ -764,7 +764,7 @@ func GetSearchFromPodcastIndex(pod *podcastindex.Podcast) *model.CommonSearchRes func UpdateSettings(downloadOnAdd bool, initialDownloadCount int, autoDownload bool, appendDateToFileName bool, appendEpisodeNumberToFileName bool, darkMode bool, downloadEpisodeImages bool, - generateNFOFile bool, dontDownloadDeletedFromDisk bool, baseUrl string, maxDownloadConcurrency int) error { + generateNFOFile bool, dontDownloadDeletedFromDisk bool, baseUrl string, maxDownloadConcurrency int, userAgent string) error { setting := db.GetOrCreateSetting() setting.AutoDownload = autoDownload @@ -778,6 +778,7 @@ func UpdateSettings(downloadOnAdd bool, initialDownloadCount int, autoDownload b setting.DontDownloadDeletedFromDisk = dontDownloadDeletedFromDisk setting.BaseUrl = baseUrl setting.MaxDownloadConcurrency = maxDownloadConcurrency + setting.UserAgent = userAgent return db.UpdateSettings(setting) } From 94b4b2b19b983c1a7c73d2e7b04e1ebf46c7031b Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 16:37:43 +1000 Subject: [PATCH 7/9] Use user-agent in all places in fileService.go and make sure user agent box is wide --- client/settings.html | 2 +- service/fileService.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/settings.html b/client/settings.html index a4922cf..6331431 100644 --- a/client/settings.html +++ b/client/settings.html @@ -110,7 +110,7 @@

Settings

diff --git a/service/fileService.go b/service/fileService.go index e2ca745..6f9c921 100644 --- a/service/fileService.go +++ b/service/fileService.go @@ -28,8 +28,7 @@ func Download(link string, episodeTitle string, podcastName string, prefix strin } client := httpClient() - req, err := http.NewRequest("GET", link, nil) - req.Header.Add("User-Agent", "") + req, err := getRequest(link) if err != nil { Logger.Errorw("Error creating request: "+link, err) } From 9942f216e110d8673adf82535c1bea34d7711271 Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 16:40:56 +1000 Subject: [PATCH 8/9] Fixing typo and formatting --- client/settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/settings.html b/client/settings.html index 6331431..3f2798b 100644 --- a/client/settings.html +++ b/client/settings.html @@ -109,7 +109,7 @@

Settings

From 7a796c481518a406377e918927bf3ac5ac2b4353 Mon Sep 17 00:00:00 2001 From: James Luck Date: Sat, 11 Jun 2022 16:45:19 +1000 Subject: [PATCH 9/9] Fixing typo --- client/settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/settings.html b/client/settings.html index 3f2798b..b349c10 100644 --- a/client/settings.html +++ b/client/settings.html @@ -109,7 +109,7 @@

Settings