From c27d8f57e281e87c88f110efe742d40ab3edcbc5 Mon Sep 17 00:00:00 2001 From: rix1337 <> Date: Sat, 30 Oct 2021 12:08:25 +0200 Subject: [PATCH 1/5] v.11.0.18 - Decrypt FC-packages last * ensures streamlined helper decryption --- .github/Changelog.md | 10 +- feedcrawler/common.py | 27 ++++- feedcrawler/db.py | 8 ++ feedcrawler/version.py | 2 +- feedcrawler/web.py | 233 +++++++++++++++++++++++------------------ 5 files changed, 170 insertions(+), 110 deletions(-) diff --git a/.github/Changelog.md b/.github/Changelog.md index 473a02e3..7f4578f6 100644 --- a/.github/Changelog.md +++ b/.github/Changelog.md @@ -9,20 +9,18 @@ --- ### Changelog: +- **11.0.18** FC-Pakete werden immer zuletzt zum Entschlüsseln angeboten - **11.0.17** Linkerkennung auf FX angepasst -- **11.0.16** Verhindere, dass - der [FeedCrawler Sponsors Helper](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) den +- **11.0.16** Verhindere, dass der [FeedCrawler Sponsors Helper](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) den selben Link mehrfach öffnet -- **11.0.15** Release der neuen Version 2.0.1 - des [FeedCrawler Sponsors Helpers](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) +- **11.0.15** Release der neuen Version 2.0.1 des [FeedCrawler Sponsors Helpers](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) - Der Helper basiert ab sofort auf Chromium, statt Firefox. - Damit werden Captchas auf DW wieder automatisch und zuverlässig gelöst. - Außerdem wurden alle Scripte aktualisiert um stabiler zu laufen. - Es sind folgende Anpassungen der Docker Konfiguration des Helpers sinnvoll: - das `--privileged`-Flag entfernen - das `--restart unless-stopped`-Flag ergänzen -- **11.0.15** Anpassung der Startseite - des [FeedCrawler Sponsors Helpers](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) +- **11.0.15** Anpassung der Startseite des [FeedCrawler Sponsors Helpers](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) - Die Seite aktualisiert ab sofort zweimalig je Minute, anstatt einmalig. - Wurde ein Captcha nicht gelöst, öffnet der Helper die Seite erneut, statt abzuwarten. - **11.0.14** Bugfixes diff --git a/feedcrawler/common.py b/feedcrawler/common.py index c2b202d4..1eb9be0d 100644 --- a/feedcrawler/common.py +++ b/feedcrawler/common.py @@ -227,10 +227,18 @@ def fullhd_title(key): def get_to_decrypt(): try: - to_decrypt = FeedDb('to_decrypt').retrieve_all_titles() + to_decrypt = FeedDb('to_decrypt').retrieve_all_titles_unordered() if to_decrypt: + easy_decrypt_exists = False + fx = CrawlerConfig('Hostnames').get('fx') + for package in to_decrypt: + if not "filecrypt." in package[1] and not fx in package[1]: + easy_decrypt_exists = True + packages = [] for package in to_decrypt: + if easy_decrypt_exists and ("filecrypt." in package[1] or fx in package[1]): + continue title = package[0] try: details = package[1].split('|') @@ -244,6 +252,23 @@ def get_to_decrypt(): 'url': url, 'password': password }) + + for package in to_decrypt: + if easy_decrypt_exists and ("filecrypt." in package[1] or fx in package[1]): + title = package[0] + try: + details = package[1].split('|') + url = details[0] + password = details[1] + except: + url = package[1] + password = "" + packages.append({ + 'name': title, + 'url': url, + 'password': password + }) + return packages else: return False diff --git a/feedcrawler/db.py b/feedcrawler/db.py index fbe70a09..8524b2dd 100644 --- a/feedcrawler/db.py +++ b/feedcrawler/db.py @@ -58,6 +58,14 @@ def retrieve_all_titles(self): items.append([str(r[0]), str(r[1])]) return items if items else None + def retrieve_all_titles_unordered(self): + res = self._conn.execute( + "SELECT distinct key, value FROM %s" % self._table) + items = [] + for r in res: + items.append([str(r[0]), str(r[1])]) + return items if items else None + def store(self, key, value): self._conn.execute("INSERT INTO '%s' VALUES ('%s', '%s')" % (self._table, key, value)) diff --git a/feedcrawler/version.py b/feedcrawler/version.py index 1358ad47..8c9b481d 100644 --- a/feedcrawler/version.py +++ b/feedcrawler/version.py @@ -10,7 +10,7 @@ def get_version(): - return "11.0.17" + return "11.0.18" def create_version_file(): diff --git a/feedcrawler/web.py b/feedcrawler/web.py index 42320070..d8f40ad5 100644 --- a/feedcrawler/web.py +++ b/feedcrawler/web.py @@ -1220,7 +1220,7 @@ def feedcrawler_sponsors_helper_sj(): // @name FeedCrawler Sponsors Helper (SJ/DJ) // @author rix1337 // @description Clicks the correct download button on SJ/DJ sub pages to speed up Click'n'Load -// @version 0.4.0 +// @version 0.5.1 // @require https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js // @match https://""" + sj + """/* // @match https://""" + dj + """/* @@ -1230,9 +1230,9 @@ def feedcrawler_sponsors_helper_sj(): // ==/UserScript== // Hier muss die von außen erreichbare Adresse des FeedCrawlers stehen (nicht bspw. die Docker-interne): -var sponsorsURL = '""" + internal.local_address + """'; +const sponsorsURL = '""" + internal.local_address + """'; // Hier kann ein Wunschhoster eingetragen werden (ohne www. und .tld): -var sponsorsHoster = ''; +const sponsorsHoster = ''; $.extend($.expr[':'], { 'containsi': function(elem, i, match, array) { @@ -1251,67 +1251,69 @@ def feedcrawler_sponsors_helper_sj(): } }); -function Sleep(milliseconds) { - return new Promise(resolve => setTimeout(resolve, milliseconds)); -} - -var tag = window.location.hash.replace("#", "").split('|'); -var title = tag[0]; -var password = tag[1]; +const tag = window.location.hash.replace("#", "").split('|'); +const title = tag[0]; +const password = tag[1]; if (title && title !== "login") { $('.wrapper').prepend('

[FeedCrawler Sponsors Helper] ' + title + '

'); $(".container").hide(); - var checkExist = setInterval(function() { - async function clickRelease() { + let i = 0; + const checkExist = setInterval(function () { + i++; if ($("tr:contains('" + title + "')").length) { $(".container").show(); $("tr:contains('" + title + "')")[0].lastChild.firstChild.click(); - if (sponsorsHelper) { - console.log("[FeedCrawler Sponsors Helper] Clicked Download button of " + title); - await Sleep(500); - var requiresLogin = $(".alert-warning").length; - if (requiresLogin) { - clearInterval(checkExist); - window.open("https://" + $(location).attr('hostname') + "#login|" + btoa(window.location)); - window.close() + if (i > 24) { + if (sponsorsHelper) { + const requiresLogin = $(".alert-warning").length; + if (requiresLogin) { + console.log("[FeedCrawler Sponsors Helper] Login required for: " + title); + clearInterval(checkExist); + window.open("https://" + $(location).attr('hostname') + "#login|" + btoa(window.location)); + window.close(); + } } + console.log("hit") + clearInterval(checkExist); + } else { + console.log("miss") } - clearInterval(checkExist); - } } - clickRelease(); + } }, 100); - if (sponsorsHelper) { - var dlExists = setInterval(async function() { - if ($("tr:contains('Download Part')").length) { - var items = $("tr:contains('Download Part')").find("a"); - var links = []; - items.each(function(index){ - links.push(items[index].href); - }) - console.log("[FeedCrawler Sponsors Helper] found download links: " + links); - clearInterval(dlExists); - window.open(sponsorsURL + '/sponsors_helper/to_download/' + btoa(links + '|' + title + '|' + password)); - window.close(); - } else if ( document.body.innerHTML.search("se das Captcha!") && !$('.center-recaptcha').length) { - if ( sponsorsHoster && $("button:containsi('" + sponsorsHoster + "')").length) { - $("button:containsi('" + sponsorsHoster + "')").click(); - } else if ( $("button:containsi('1fichier')").length) { - $("button:containsi('1fichier')").click(); - } else if ( $("button:containsi('ddownload')").length) { - $("button:containsi('ddownload')").click(); - } else if ( $("button:containsi('turbo')").length) { - $("button:containsi('turbo')").click(); - } else if ( $("button:containsi('filer')").length) { - $("button:containsi('filer')").click(); - } else { - $("div.modal-body").find("button.btn.btn-secondary.btn-block").click(); - } - console.log("[FeedCrawler Sponsors Helper] Clicked Download button to trigger reCAPTCHA"); - } - }, 100); - } -}; + let j = 0; + let dl = false; + const dlExists = setInterval(function () { + j++; + if ($("tr:contains('Download Part')").length) { + const items = $("tr:contains('Download Part')").find("a"); + const links = []; + items.each(function (index) { + links.push(items[index].href); + }); + console.log("[FeedCrawler Sponsors Helper] found download links: " + links); + clearInterval(dlExists); + window.open(sponsorsURL + '/sponsors_helper/to_download/' + btoa(links + '|' + title + '|' + password)); + window.close(); + } else if (j > 24 && !dl) { + if (sponsorsHoster && $("button:containsi('" + sponsorsHoster + "')").length) { + $("button:containsi('" + sponsorsHoster + "')").click(); + } else if ($("button:containsi('1fichier')").length) { + $("button:containsi('1fichier')").click(); + } else if ($("button:containsi('ddownload')").length) { + $("button:containsi('ddownload')").click(); + } else if ($("button:containsi('turbo')").length) { + $("button:containsi('turbo')").click(); + } else if ($("button:containsi('filer')").length) { + $("button:containsi('filer')").click(); + } else { + $("div.modal-body").find("button.btn.btn-secondary.btn-block").click(); + } + console.log("[FeedCrawler Sponsors Helper] Clicked Download button to trigger reCAPTCHA"); + dl = true; + } + }, 100); +} """, 200 except: return "Failed", 400 @@ -1329,74 +1331,101 @@ def feedcrawler_sponsors_helper_fc(): // @name FeedCrawler Sponsors Helper (FC) // @author rix1337 // @description Forwards Click'n'Load to FeedCrawler -// @version 0.5.0 +// @version 0.5.1 // @match *.filecrypt.cc/* // @match *.filecrypt.co/* // @grant window.close // ==/UserScript== // Hier muss die von außen erreichbare Adresse des FeedCrawlers stehen (nicht bspw. die Docker-interne): -var sponsorsURL = '""" + internal.local_address + """'; +const sponsorsURL = '""" + internal.local_address + """'; // Hier kann ein Wunschhoster eingetragen werden (ohne www. und .tld): -var sponsorsHoster = ''; +const sponsorsHoster = ''; -var tag = window.location.hash.replace("#", "").split('|'); -var title = tag[0] -var password = tag[1] -var ids = tag[2] -var urlParams = new URLSearchParams(window.location.search); +const tag = window.location.hash.replace("#", "").split('|'); +const title = tag[0]; +const password = tag[1]; +const ids = tag[2]; +const urlParams = new URLSearchParams(window.location.search); function Sleep(milliseconds) { return new Promise(resolve => setTimeout(resolve, milliseconds)); } - var mirrorsAvailable = false; - try { - mirrorsAvailable = document.querySelector('.mirror').querySelectorAll("a"); - } catch {} - var cnlAllowed = false; +const fx = (document.getElementById("customlogo").getAttribute('src') === '/css/custom/f38ed.png'); + +const checkFxPass = setInterval(function () { + if (document.getElementById("p4assw0rt")) { + if (fx) { + const pw = atob('ZnVueGQ='); + console.log("[FeedCrawler Sponsors Helper] entering Password: " + pw); + document.getElementById("p4assw0rt").value = pw; + document.getElementById("p4assw0rt").parentNode.nextElementSibling.click(); + } + clearInterval(checkFxPass); + } +}, 100); -if (mirrorsAvailable && sponsorsHoster) { - const currentURL = window.location.href; - var desiredMirror = ""; - var i; - for (i = 0; i < mirrorsAvailable.length; i++) { - if (mirrorsAvailable[i].text.includes(sponsorsHoster)) { - var ep = ""; - var cur_ep = urlParams.get('episode'); - if (cur_ep) { - ep = "&episode=" + cur_ep; - } - desiredMirror = mirrorsAvailable[i].href + ep + window.location.hash; - } - } - if (desiredMirror) { - if (!currentURL.includes(desiredMirror)) { - console.log("[FeedCrawler Sponsors Helper] switching to desired Mirror: " + sponsorsHoster); - window.location = desiredMirror; - } else { - console.log("[FeedCrawler Sponsors Helper] already at the desired Mirror: " + sponsorsHoster); - cnlAllowed = true; - } - } else { - console.log("[FeedCrawler Sponsors Helper] desired Mirror not available: " + sponsorsHoster); - cnlAllowed = true; - } +const checkSFPass = setInterval(function () { + if (document.getElementById("p4assw0rt") && !fx) { + const pw = atob('c2VyaWVuZmFucy5vcmc='); + console.log("[FeedCrawler Sponsors Helper] entering Password: " + pw); + document.getElementById("p4assw0rt").value = pw; + document.getElementById("p4assw0rt").parentNode.nextElementSibling.click(); + clearInterval(checkSFPass); + } +}, 100); + + +let mirrorsAvailable = false; +try { + mirrorsAvailable = document.querySelector('.mirror').querySelectorAll("a"); +} catch { +} +let cnlAllowed = false; + +if (mirrorsAvailable && sponsorsHoster) { + const currentURL = window.location.href; + let desiredMirror = ""; + let i; + for (i = 0; i < mirrorsAvailable.length; i++) { + if (mirrorsAvailable[i].text.includes(sponsorsHoster)) { + let ep = ""; + const cur_ep = urlParams.get('episode'); + if (cur_ep) { + ep = "&episode=" + cur_ep; + } + desiredMirror = mirrorsAvailable[i].href + ep + window.location.hash; + } + } + + if (desiredMirror) { + if (!currentURL.toLowerCase().includes(desiredMirror.toLowerCase())) { + console.log("[FeedCrawler Sponsors Helper] switching to desired Mirror: " + sponsorsHoster); + window.location = desiredMirror; + } else { + console.log("[FeedCrawler Sponsors Helper] already at the desired Mirror: " + sponsorsHoster); + cnlAllowed = true; + } + } else { + console.log("[FeedCrawler Sponsors Helper] desired Mirror not available: " + sponsorsHoster); + cnlAllowed = true; + } } else { - cnlAllowed = true; + cnlAllowed = true; } -var cnlExists = setInterval(async function() { - if (cnlAllowed && document.getElementsByClassName("cnlform").length) { - clearInterval(cnlExists); - document.getElementById("cnl_btn").click(); - console.log("[FeedCrawler Sponsors Helper] attempting Click'n'Load"); - await Sleep(4000); - window.close(); - } +const cnlExists = setInterval(async function () { + if (cnlAllowed && document.getElementsByClassName("cnlform").length) { + clearInterval(cnlExists); + document.getElementById("cnl_btn").click(); + console.log("[FeedCrawler Sponsors Helper] attempting Click'n'Load"); + await Sleep(30000); + window.close(); + } }, 100); """, 200 except: From 93bafe83c99091283de1d10d2cfa9fa1e70f318b Mon Sep 17 00:00:00 2001 From: rix1337 <> Date: Sat, 30 Oct 2021 12:10:00 +0200 Subject: [PATCH 2/5] v.11.0.18 - Refresh web interface less often --- .github/Changelog.md | 1 + feedcrawler/web/js/feedcrawler.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/Changelog.md b/.github/Changelog.md index 7f4578f6..f76c4731 100644 --- a/.github/Changelog.md +++ b/.github/Changelog.md @@ -10,6 +10,7 @@ ### Changelog: - **11.0.18** FC-Pakete werden immer zuletzt zum Entschlüsseln angeboten +- **11.0.18** Lade Daten im Webinterface seltener neu (verbessert Performance) - **11.0.17** Linkerkennung auf FX angepasst - **11.0.16** Verhindere, dass der [FeedCrawler Sponsors Helper](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) den selben Link mehrfach öffnet diff --git a/feedcrawler/web/js/feedcrawler.js b/feedcrawler/web/js/feedcrawler.js index 4c3f6be1..83b68927 100644 --- a/feedcrawler/web/js/feedcrawler.js +++ b/feedcrawler/web/js/feedcrawler.js @@ -885,7 +885,7 @@ app.controller('crwlCtrl', function ($scope, $http, $timeout) { } } $scope.checkMyJD(); - }, 5000) + }, 15000) }; $scope.checkMyJD(); @@ -896,7 +896,7 @@ app.controller('crwlCtrl', function ($scope, $http, $timeout) { getLog(); } $scope.updateLog(); - }, 5000) + }, 15000) }; $scope.updateLog(); @@ -907,7 +907,7 @@ app.controller('crwlCtrl', function ($scope, $http, $timeout) { getCrawlTimes(); } $scope.updateCrawlTimes(); - }, 5000) + }, 15000) }; $scope.updateCrawlTimes(); From b2368e08b33475b0c24df002254adf28ef640659 Mon Sep 17 00:00:00 2001 From: rix1337 <> Date: Sat, 30 Oct 2021 12:15:44 +0200 Subject: [PATCH 3/5] v.11.0.18 - Close #543 * catch ombi requests without imdb id --- feedcrawler/ombi.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/feedcrawler/ombi.py b/feedcrawler/ombi.py index e718c252..55be60ab 100644 --- a/feedcrawler/ombi.py +++ b/feedcrawler/ombi.py @@ -27,7 +27,10 @@ def imdb_movie(imdb_id): year = str(output.data['year']) return title + " " + year except: - print(u"[Ombi] - Fehler beim Abruf der IMDb für: " + imdb_id) + if imdb_id is None: + internal.logger.debug("Ein Film ohne IMDb-ID wurde angefordert.") + else: + print(u"[Ombi] - Fehler beim Abruf der IMDb für: " + imdb_id) return False, False @@ -49,7 +52,10 @@ def imdb_show(imdb_id): return title, eps except: - print(u"[Ombi] - Fehler beim Abruf der IMDb für: " + imdb_id) + if imdb_id is None: + internal.logger.debug("Eine Serie ohne IMDb-ID wurde angefordert.") + else: + print(u"[Ombi] - Fehler beim Abruf der IMDb für: " + imdb_id) return False, False, False From b27c03c441c74423bb06fceeeae1a79d6dd2c2d2 Mon Sep 17 00:00:00 2001 From: rix1337 <> Date: Sat, 30 Oct 2021 12:16:54 +0200 Subject: [PATCH 4/5] v.11.0.18 - Close #543 * catch ombi requests without imdb id --- .github/Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/Changelog.md b/.github/Changelog.md index f76c4731..e7daa935 100644 --- a/.github/Changelog.md +++ b/.github/Changelog.md @@ -11,6 +11,7 @@ ### Changelog: - **11.0.18** FC-Pakete werden immer zuletzt zum Entschlüsseln angeboten - **11.0.18** Lade Daten im Webinterface seltener neu (verbessert Performance) +- **11.0.18** Verhindere Crash bei Ombi-Anfragen ohne IMDb-ID #543 Danke @postboy99 - **11.0.17** Linkerkennung auf FX angepasst - **11.0.16** Verhindere, dass der [FeedCrawler Sponsors Helper](https://github.com/rix1337/RSScrawler/wiki/5.-FeedCrawler-Sponsors-Helper) den selben Link mehrfach öffnet From 3a7e9f7bfa8fcab59f3b8b054a1f333d33d7fe22 Mon Sep 17 00:00:00 2001 From: rix1337 <> Date: Sat, 30 Oct 2021 13:24:20 +0200 Subject: [PATCH 5/5] v.11.0.18 - Fix FX feed and web search --- .github/Changelog.md | 1 + feedcrawler/search/shared/content_all.py | 1 + feedcrawler/sites/content_all_fx.py | 4 ++-- feedcrawler/sites/shared/content_all.py | 12 +++++++----- feedcrawler/sites/shared/internal_feed.py | 12 +++++++----- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/Changelog.md b/.github/Changelog.md index e7daa935..d8ead0a7 100644 --- a/.github/Changelog.md +++ b/.github/Changelog.md @@ -9,6 +9,7 @@ --- ### Changelog: +- **11.0.18** FX Suche (Feed und Web) wiederhergestellt - **11.0.18** FC-Pakete werden immer zuletzt zum Entschlüsseln angeboten - **11.0.18** Lade Daten im Webinterface seltener neu (verbessert Performance) - **11.0.18** Verhindere Crash bei Ombi-Anfragen ohne IMDb-ID #543 Danke @postboy99 diff --git a/feedcrawler/search/shared/content_all.py b/feedcrawler/search/shared/content_all.py index 908f71cd..496b698a 100644 --- a/feedcrawler/search/shared/content_all.py +++ b/feedcrawler/search/shared/content_all.py @@ -137,6 +137,7 @@ def download(payload): for hoster in hosters: url_hosters.append(['https://' + nk + hoster["href"], hoster.text]) elif "FX" in site: + download_method = add_decrypt_instead_of_download key = payload[1] password = payload[2] else: diff --git a/feedcrawler/sites/content_all_fx.py b/feedcrawler/sites/content_all_fx.py index ef9c9f06..dba6dd5a 100644 --- a/feedcrawler/sites/content_all_fx.py +++ b/feedcrawler/sites/content_all_fx.py @@ -5,7 +5,7 @@ import feedcrawler.sites.shared.content_all as shared_blogs from feedcrawler.config import CrawlerConfig from feedcrawler.db import FeedDb -from feedcrawler.myjd import myjd_download +from feedcrawler.sites.shared.internal_feed import add_decrypt_instead_of_download from feedcrawler.sites.shared.internal_feed import fx_feed_enricher from feedcrawler.sites.shared.internal_feed import fx_get_download_links from feedcrawler.url import get_url @@ -64,7 +64,7 @@ def __init__(self, filename): self.get_url_method = get_url self.get_url_headers_method = get_url_headers self.get_download_links_method = fx_get_download_links - self.download_method = myjd_download + self.download_method = add_decrypt_instead_of_download try: self.imdb = float(self.config.get('imdb')) diff --git a/feedcrawler/sites/shared/content_all.py b/feedcrawler/sites/shared/content_all.py index 3d700c94..3c999cb1 100644 --- a/feedcrawler/sites/shared/content_all.py +++ b/feedcrawler/sites/shared/content_all.py @@ -365,7 +365,7 @@ def download_hevc(self, title): for result in search_results: i += 1 - key = result[0] + key = result[0].replace(" ", ".") if feedsearch_title in key: payload = result[1].split("|") @@ -387,7 +387,7 @@ def download_hevc(self, title): link = get_url(link) link_grabbed = True get_download_links_method = fx_get_download_links - download_method = myjd_download + download_method = add_decrypt_instead_of_download elif "NK" in site: get_download_links_method = nk_page_download_link download_method = myjd_download @@ -502,7 +502,7 @@ def download_dual_language(self, title, hevc=False): hevc_found = False for result in search_results: - key = result[0] + key = result[0].replace(" ", ".") if feedsearch_title in key and ".dl." in key.lower() and (hevc and is_hevc(key)): hevc_found = True @@ -510,7 +510,7 @@ def download_dual_language(self, title, hevc=False): for result in search_results: i += 1 - key = result[0] + key = result[0].replace(" ", ".") if feedsearch_title in key: payload = result[1].split("|") @@ -529,7 +529,7 @@ def download_dual_language(self, title, hevc=False): elif "FX" in site: link = get_url(link) get_download_links_method = fx_get_download_links - download_method = myjd_download + download_method = add_decrypt_instead_of_download elif "NK" in site: get_download_links_method = nk_page_download_link download_method = myjd_download @@ -605,6 +605,7 @@ def download_dual_language(self, title, hevc=False): def download_imdb(self, key, download_links, score, imdb_id, hevc_retail, site, download_method): + key = key.replace(" ", ".") added_items = [] if not hevc_retail: if self.hevc_retail: @@ -677,6 +678,7 @@ def download_imdb(self, key, download_links, score, imdb_id, hevc_retail, site, def download_feed(self, key, content, hevc_retail): + key = key.replace(" ", ".") added_items = [] if not hevc_retail: if self.hevc_retail: diff --git a/feedcrawler/sites/shared/internal_feed.py b/feedcrawler/sites/shared/internal_feed.py index bffd26e9..73dfd20b 100644 --- a/feedcrawler/sites/shared/internal_feed.py +++ b/feedcrawler/sites/shared/internal_feed.py @@ -513,15 +513,16 @@ def fx_get_download_links(self, content, title): def fx_feed_enricher(feed): feed = BeautifulSoup(feed, 'lxml') + fx = CrawlerConfig('Hostnames').get('fx') articles = feed.findAll("article") entries = [] for article in articles: try: article = BeautifulSoup(str(article), 'lxml') - titles = article.findAll("a", href=re.compile("filecrypt")) + titles = article.findAll("a", href=re.compile("(filecrypt|safe." + fx + ")")) for title in titles: - title = title.text.encode("ascii", errors="ignore").decode().replace("/", "") + title = title.text.encode("ascii", errors="ignore").decode().replace("/", "").replace(" ", ".") if title: if "download" in title.lower(): try: @@ -550,6 +551,7 @@ def fx_feed_enricher(feed): def fx_search_results(content): + fx = CrawlerConfig('Hostnames').get('fx') articles = content.find("main").find_all("article") result_urls = [] for article in articles: @@ -566,11 +568,11 @@ def fx_search_results(content): for result in results: article = BeautifulSoup(str(result), 'lxml') - titles = article.find_all("a", href=re.compile("filecrypt")) + titles = article.find_all("a", href=re.compile("(filecrypt|safe." + fx + ")")) for title in titles: link = article.find("link", rel="canonical")["href"] - title = title.text.encode("ascii", errors="ignore").decode().replace("/", "") - if title: + title = title.text.encode("ascii", errors="ignore").decode().replace("/", "").replace(" ", ".") + if title and "-fun" in title.lower(): if "download" in title.lower(): try: title = str(content.find("strong", text=re.compile(r".*Release.*")).nextSibling)