From 567916a3b5f4e1f78810c102b38a9b5889e36e08 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 23 Jul 2024 10:04:07 -0400 Subject: [PATCH 1/3] fix icon and email content --- mastodon/models/email.py | 2 +- social/templates/feed_events.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mastodon/models/email.py b/mastodon/models/email.py index 3e4e029a..19bc753b 100644 --- a/mastodon/models/email.py +++ b/mastodon/models/email.py @@ -46,7 +46,7 @@ def _send(email, subject, body): def generate_login_email(email: str, action: str) -> tuple[str, str]: if action != "verify": account = EmailAccount.objects.filter(handle__iexact=email).first() - action = "register" if account and account.user else "login" + action = "login" if account and account.user else "register" s = {"e": email, "a": action} # v = TimestampSigner().sign_object(s) code = b62_encode(random.randint(pow(62, 4), pow(62, 5) - 1)) diff --git a/social/templates/feed_events.html b/social/templates/feed_events.html index b59b4dcb..b446a7be 100644 --- a/social/templates/feed_events.html +++ b/social/templates/feed_events.html @@ -27,7 +27,8 @@
- @{{ post.author.handle }} + @{{ post.author.handle }}
From 8a9b5bfab7b2171a6b4d74041ca167919a6dade1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 23 Jul 2024 10:36:01 -0400 Subject: [PATCH 2/3] fix a few edge cases --- catalog/common/models.py | 2 +- catalog/common/sites.py | 20 ++++++++++---------- mastodon/models/email.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/catalog/common/models.py b/catalog/common/models.py index 4a7c1b1e..1215b542 100644 --- a/catalog/common/models.py +++ b/catalog/common/models.py @@ -654,7 +654,7 @@ def display_description(self) -> str: @property def brief_description(self): - return (self.display_description or "")[:155] + return (str(self.display_description) or "")[:155] @classmethod def get_by_url(cls, url_or_b62: str, resolve_merge=False) -> "Self | None": diff --git a/catalog/common/sites.py b/catalog/common/sites.py index 9821face..34a56407 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -131,18 +131,18 @@ def match_existing_item_for_resource( ids = resource.get_lookup_ids(cls.DEFAULT_MODEL) for t, v in ids: matched = None + # matched = model.objects.filter( + # primary_lookup_id_type=t, + # primary_lookup_id_value=v, + # title=resource.metadata["title"], + # ).first() + # if matched is None and resource.id_type not in [ + # IdType.DoubanMusic, # DoubanMusic has many dirty data with same UPC + # # IdType.Goodreads, # previous scraper generated some dirty data + # ]: matched = model.objects.filter( - primary_lookup_id_type=t, - primary_lookup_id_value=v, - title=resource.metadata["title"], + primary_lookup_id_type=t, primary_lookup_id_value=v ).first() - if matched is None and resource.id_type not in [ - IdType.DoubanMusic, # DoubanMusic has many dirty data with same UPC - # IdType.Goodreads, # previous scraper generated some dirty data - ]: - matched = model.objects.filter( - primary_lookup_id_type=t, primary_lookup_id_value=v - ).first() if matched is None: matched = model.objects.filter( primary_lookup_id_type=resource.id_type, diff --git a/mastodon/models/email.py b/mastodon/models/email.py index 19bc753b..2401e539 100644 --- a/mastodon/models/email.py +++ b/mastodon/models/email.py @@ -40,7 +40,7 @@ def _send(email, subject, body): fail_silently=False, ) except Exception as e: - logger.error(f"send email to {email} failed: {e}", extra={"exception": e}) + logger.error(f"send email to {email} failed: {e}") @staticmethod def generate_login_email(email: str, action: str) -> tuple[str, str]: From 4c299ea4affe216763eba87eaedfa0f8e8bbe829 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 23 Jul 2024 10:46:31 -0400 Subject: [PATCH 3/3] fix edge case --- catalog/sites/imdb.py | 14 +++++--------- catalog/sites/tmdb.py | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/catalog/sites/imdb.py b/catalog/sites/imdb.py index 66a07111..faf09b56 100644 --- a/catalog/sites/imdb.py +++ b/catalog/sites/imdb.py @@ -32,11 +32,7 @@ def id_to_url(cls, id_value): return "https://www.imdb.com/title/" + id_value + "/" def scrape(self): - res_data = {} - try: - res_data = search_tmdb_by_imdb_id(self.id_value) - except: - pass + res_data = search_tmdb_by_imdb_id(self.id_value) url = None pd = None @@ -91,15 +87,15 @@ def scrape_imdb(self): "genre": ( [x["text"] for x in d["genres"]["genres"]] if d.get("genres") else [] ), - "brief": ( - d["plot"].get("plotText").get("plainText") if d.get("plot") else None - ), + "brief": d.get("plot", {}).get("plotText", {}).get("plainText", ""), "cover_image_url": ( d["primaryImage"].get("url") if d.get("primaryImage") else None ), } data["localized_title"] = [{"lang": "en", "text": data["title"]}] - data["localized_description"] = [{"lang": "en", "text": data["brief"]}] + data["localized_description"] = ( + [{"lang": "en", "text": data["brief"]}] if data["brief"] else [] + ) if d.get("series"): episode_info = d["series"].get("episodeNumber") if episode_info: diff --git a/catalog/sites/tmdb.py b/catalog/sites/tmdb.py index 26b86d6e..8af042ec 100644 --- a/catalog/sites/tmdb.py +++ b/catalog/sites/tmdb.py @@ -37,11 +37,11 @@ def _get_language_code(): def _get_preferred_languages(): langs = {} - for l in PREFERRED_LANGUAGES: - if l == "zh": + for lang in PREFERRED_LANGUAGES: + if lang == "zh": langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW", "zh-hk": "zh-HK"}) else: - langs[l] = l + langs[lang] = lang return langs @@ -50,9 +50,13 @@ def _get_preferred_languages(): def search_tmdb_by_imdb_id(imdb_id): + if not settings.TMDB_API3_KEY or settings.TMDB_API3_KEY == "TESTONLY": + return {} tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&external_source=imdb_id" - res_data = BasicDownloader(tmdb_api_url).download().json() - return res_data + try: + return BasicDownloader(tmdb_api_url).download().json() + except Exception: + return {} def query_tmdb_tv_episode(tv, season, episode): @@ -126,7 +130,7 @@ def scrape(self): actor = list(map(lambda x: x["name"], res_data["credits"]["cast"])) area = [] - other_info = {} + # other_info = {} # other_info['TMDB评分'] = res_data['vote_average'] # other_info['分级'] = res_data['contentRating'] # other_info['Metacritic评分'] = res_data['metacriticRating'] @@ -233,9 +237,9 @@ def scrape(self): ) actor = list(map(lambda x: x["name"], res_data["credits"]["cast"])) area = [] - other_info = {} - other_info["Seasons"] = res_data["number_of_seasons"] - other_info["Episodes"] = res_data["number_of_episodes"] + # other_info = {} + # other_info["Seasons"] = res_data["number_of_seasons"] + # other_info["Episodes"] = res_data["number_of_episodes"] # TODO: use GET /configuration to get base url img_url = ( ("https://image.tmdb.org/t/p/original/" + res_data["poster_path"]) @@ -446,7 +450,7 @@ def scrape(self): season_id = v[1] episode_id = v[2] site = TMDB_TV(TMDB_TV.id_to_url(show_id)) - show_resource = site.get_resource_ready(auto_create=False, auto_link=False) + site.get_resource_ready(auto_create=False, auto_link=False) api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits" d = BasicDownloader(api_url).download().json() if not d.get("id"):