more data checks in scrapers

catalog/sites/bangumi.py

@@ -106,7 +106,9 @@ def scrape(self):
             [title] + (other_title or []) + ([orig_title] if orig_title else [])
         )
         localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
-        localized_desc = [{"lang": detect_language(brief), "text": brief}]
+        localized_desc = (
+            [{"lang": detect_language(brief), "text": brief}] if brief else []
+        )
         data = {
             "localized_title": localized_title,
             "localized_description": localized_desc,

catalog/sites/bgg.py

@@ -56,7 +56,9 @@ def scrape(self):
         pd = ResourceContent(
             metadata={
                 "localized_title": localized_title,
-                "localized_description": [{"lang": "en", "text": brief}],
+                "localized_description": (
+                    [{"lang": "en", "text": brief}] if brief else []
+                ),
                 "title": title,
                 "other_title": other_title,
                 "genre": category,

catalog/sites/douban_book.py

@@ -190,7 +190,7 @@ def scrape(self):
             "subtitle": subtitle,
             "localized_title": [{"lang": lang, "text": title}],
             "localized_subtitle": [{"lang": lang, "text": subtitle}],
-            "localized_description": [{"lang": lang, "text": brief}],
+            "localized_description": [{"lang": lang, "text": brief}] if brief else [],
             "orig_title": orig_title,
             "author": authors,
             "translator": translators,

catalog/sites/douban_game.py

@@ -92,7 +92,9 @@ def scrape(self):
 
         titles = uniq([title] + other_title + ([orig_title] if orig_title else []))
         localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
-        localized_desc = [{"lang": detect_language(brief), "text": brief}]
+        localized_desc = (
+            [{"lang": detect_language(brief), "text": brief}] if brief else []
+        )
 
         pd = ResourceContent(
             metadata={

catalog/sites/douban_movie.py

@@ -212,7 +212,9 @@ def scrape(self):
             + (other_title if other_title else [])
         )
         localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
-        localized_desc = [{"lang": detect_language(brief), "text": brief}]
+        localized_desc = (
+            [{"lang": detect_language(brief), "text": brief}] if brief else []
+        )
         pd = ResourceContent(
             metadata={
                 "title": title,

catalog/sites/douban_music.py

@@ -90,7 +90,7 @@ def scrape(self):
         data = {
             "title": title,
             "localized_title": localized_title,
-            "localized_description": [{"lang": lang, "text": brief}],
+            "localized_description": [{"lang": lang, "text": brief}] if brief else [],
             "artist": artist,
             "genre": genre,
             "release_date": release_date,

catalog/sites/goodreads.py

@@ -73,7 +73,9 @@ def scrape(self, response=None):
         lang = detect_language(b["title"] + " " + (b["description"] or ""))
         data["localized_title"] = [{"lang": lang, "text": b["title"]}]
         data["localized_subtitle"] = []  # Goodreads does not support subtitle
-        data["localized_description"] = [{"lang": lang, "text": b["description"]}]
+        data["localized_description"] = (
+            [{"lang": lang, "text": b["description"]}] if b["description"] else []
+        )
 
         if data["brief"]:
             data["brief"] = re.sub(

catalog/sites/rss.py

@@ -88,15 +88,17 @@ def scrape(self):
         feed = self.parse_feed_from_url(self.url)
         if not feed:
             raise ValueError(f"no feed avaialble in {self.url}")
-        title = feed["title"]
+        title = feed["title"].strip()
+        if not title:
+            raise ParseError(self, "title")
         desc = html_to_text(feed["description"])
         lang = detect_language(title + " " + desc)
         pd = ResourceContent(
             metadata={
                 "title": title,
                 "brief": desc,
                 "localized_title": [{"lang": lang, "text": title}],
-                "localized_description": [{"lang": lang, "text": desc}],
+                "localized_description": [{"lang": lang, "text": desc}] if desc else [],
                 "host": (
                     [feed.get("itunes_author")] if feed.get("itunes_author") else []
                 ),