More fix on Goodreads and bangumi

boofilsic/settings.py

@@ -84,9 +84,13 @@
     # Number of marks required for an item to be included in discover
     NEODB_MIN_MARKS_FOR_DISCOVER=(int, 1),
     # if True, only show title language with NEODB_PREFERRED_LANGUAGES
-    NEODB_FILTER_LANGUAGE_FOR_DISCOVER=(bool, False),
+    NEODB_DISCOVER_FILTER_LANGUAGE=(bool, False),
     # if True, only show items marked by local users rather than entire network
-    NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER=(bool, False),
+    NEODB_DISCOVER_SHOW_LOCAL_ONLY=(bool, False),
+    # if True, show popular public posts instead of recent ones.
+    NEODB_DISCOVER_SHOW_POPULAR_POSTS=(bool, False),
+    # update popular items every X minutes.
+    NEODB_DISCOVER_UPDATE_INTERVAL=(int, 60),
     # Disable cron jobs, * for all
     NEODB_DISABLE_CRON_JOBS=(list, []),
     # federated search peers
@@ -219,8 +223,10 @@
 
 MIN_MARKS_FOR_DISCOVER = env("NEODB_MIN_MARKS_FOR_DISCOVER")
 
-FILTER_LANGUAGE_FOR_DISCOVER = env("NEODB_FILTER_LANGUAGE_FOR_DISCOVER")
-FILTER_LOCAL_ONLY_FOR_DISCOVER = env("NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER")
+DISCOVER_UPDATE_INTERVAL = env("NEODB_DISCOVER_UPDATE_INTERVAL")
+DISCOVER_FILTER_LANGUAGE = env("NEODB_DISCOVER_FILTER_LANGUAGE")
+DISCOVER_SHOW_LOCAL_ONLY = env("NEODB_DISCOVER_SHOW_LOCAL_ONLY")
+DISCOVER_SHOW_POPULAR_POSTS = env("NEODB_DISCOVER_SHOW_POPULAR_POSTS")
 
 MASTODON_ALLOWED_SITES = env("NEODB_LOGIN_MASTODON_WHITELIST")
 

catalog/book/models.py

@@ -173,7 +173,7 @@ class Edition(Item):
     )
     language = LanguageListField()
     pub_house = jsondata.CharField(
-        _("publishing house"), null=True, blank=False, max_length=500
+        _("publishing house"), null=True, blank=True, max_length=500
     )
     pub_year = jsondata.IntegerField(
         _("publication year"),

catalog/common/models.py

@@ -419,6 +419,14 @@ class Meta:
             ]
         ]
 
+    def can_soft_delete(self):
+        return (
+            not self.is_deleted
+            and not self.merged_to_item_id
+            and not self.merged_from_items.exists()
+            and not self.child_items.exists()
+        )
+
     def delete(
         self,
         using: Any = None,

catalog/jobs/discover.py

@@ -31,17 +31,17 @@
 
 @JobManager.register
 class DiscoverGenerator(BaseJob):
-    interval = timedelta(hours=1)
+    interval = timedelta(minutes=settings.DISCOVER_UPDATE_INTERVAL)
 
     def get_popular_marked_item_ids(self, category, days, exisiting_ids):
         qs = (
             ShelfMember.objects.filter(q_item_in_category(category))
             .filter(created_time__gt=timezone.now() - timedelta(days=days))
             .exclude(item_id__in=exisiting_ids)
         )
-        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+        if settings.DISCOVER_SHOW_LOCAL_ONLY:
             qs = qs.filter(local=True)
-        if settings.FILTER_LANGUAGE_FOR_DISCOVER:
+        if settings.DISCOVER_FILTER_LANGUAGE:
             q = None
             for loc in PREFERRED_LOCALES:
                 if q:
@@ -63,7 +63,7 @@ def get_popular_commented_podcast_ids(self, days, exisiting_ids):
         qs = Comment.objects.filter(q_item_in_category(ItemCategory.Podcast)).filter(
             created_time__gt=timezone.now() - timedelta(days=days)
         )
-        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+        if settings.DISCOVER_SHOW_LOCAL_ONLY:
             qs = qs.filter(local=True)
         return list(
             qs.annotate(p=F("item__podcastepisode__program"))
@@ -87,7 +87,7 @@ def cleanup_shows(self, items):
 
     def run(self):
         logger.info("Discover data update start.")
-        local = settings.FILTER_LOCAL_ONLY_FOR_DISCOVER
+        local = settings.DISCOVER_SHOW_LOCAL_ONLY
         gallery_categories = [
             ItemCategory.Book,
             ItemCategory.Movie,
@@ -165,36 +165,39 @@ def run(self):
         tags = TagManager.popular_tags(days=14, local_only=local)[:40]
         excluding_identities = Takahe.get_no_discover_identities()
 
-        reviews = (
-            Review.objects.filter(visibility=0)
-            .exclude(owner_id__in=excluding_identities)
-            .order_by("-created_time")
-        )
-        if local:
-            reviews = reviews.filter(local=True)
-        post_ids = (
-            set(
-                Takahe.get_popular_posts(
-                    28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
-                ).values_list("pk", flat=True)[:5]
-            )
-            | set(
-                Takahe.get_popular_posts(
-                    14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
-                ).values_list("pk", flat=True)[:5]
+        if settings.NEODB_DISCOVER_SHOW_POPULAR_POSTS:
+            reviews = (
+                Review.objects.filter(visibility=0)
+                .exclude(owner_id__in=excluding_identities)
+                .order_by("-created_time")
             )
-            | set(
-                Takahe.get_popular_posts(
-                    7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
-                ).values_list("pk", flat=True)[:10]
-            )
-            | set(
-                Takahe.get_popular_posts(1, 0, excluding_identities, local).values_list(
-                    "pk", flat=True
-                )[:3]
+            if local:
+                reviews = reviews.filter(local=True)
+            post_ids = (
+                set(
+                    Takahe.get_popular_posts(
+                        28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                    ).values_list("pk", flat=True)[:5]
+                )
+                | set(
+                    Takahe.get_popular_posts(
+                        14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                    ).values_list("pk", flat=True)[:5]
+                )
+                | set(
+                    Takahe.get_popular_posts(
+                        7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                    ).values_list("pk", flat=True)[:10]
+                )
+                | set(
+                    Takahe.get_popular_posts(
+                        1, 0, excluding_identities, local
+                    ).values_list("pk", flat=True)[:3]
+                )
+                | set(reviews.values_list("posts", flat=True)[:5])
             )
-            | set(reviews.values_list("posts", flat=True)[:5])
-        )
+        else:
+            post_ids = []
         cache.set("public_gallery", gallery_list, timeout=None)
         cache.set("trends_links", trends, timeout=None)
         cache.set("featured_collections", collection_ids, timeout=None)

catalog/podcast/models.py

@@ -17,7 +17,7 @@
     PrimaryLookupIdDescriptor,
     jsondata,
 )
-from catalog.common.models import LanguageListField
+from catalog.common.models import LIST_OF_ONE_PLUS_STR_SCHEMA, LanguageListField
 
 
 class PodcastInSchema(ItemInSchema):
@@ -52,10 +52,12 @@ class Podcast(Item):
 
     language = LanguageListField()
 
-    host = jsondata.ArrayField(
+    host = jsondata.JSONField(
         verbose_name=_("host"),
-        base_field=models.CharField(blank=True, default="", max_length=200),
+        null=False,
+        blank=False,
         default=list,
+        schema=LIST_OF_ONE_PLUS_STR_SCHEMA,
     )
 
     official_site = jsondata.CharField(
@@ -97,6 +99,14 @@ def feed_url(self):
     def child_items(self):
         return self.episodes.filter(is_deleted=False, merged_to_item=None)
 
+    def can_soft_delete(self):
+        # override can_soft_delete() and allow delete podcast with episodes
+        return (
+            not self.is_deleted
+            and not self.merged_to_item_id
+            and not self.merged_from_items.exists()
+        )
+
 
 class PodcastEpisode(Item):
     category = ItemCategory.Podcast

catalog/sites/bangumi.py

@@ -75,7 +75,11 @@ def scrape(self):
             v = i["value"]
             match k:
                 case "别名":
-                    other_title = [d["v"] for d in v] if type(v) == list else v
+                    other_title = (
+                        [d["v"] for d in v]
+                        if type(v) == list
+                        else ([v] if isinstance(v, str) else [])
+                    )
                 case "imdb_id":
                     imdb_code = v
                 case "isbn":
@@ -87,12 +91,20 @@ def scrape(self):
                 case "导演":
                     director = v
                 case "作者":
-                    authors = [d["v"] for d in v] if type(v) == list else v
+                    authors = (
+                        [d["v"] for d in v]
+                        if type(v) == list
+                        else ([v] if isinstance(v, str) else [])
+                    )
                 case "平台":
-                    platform = [d["v"] for d in v] if type(v) == list else v
+                    platform = (
+                        [d["v"] for d in v]
+                        if type(v) == list
+                        else ([v] if isinstance(v, str) else [])
+                    )
                 case "游戏类型":
                     genre = (
-                        [i["v"] for i in v]
+                        [d["v"] for d in v]
                         if isinstance(v, list)
                         else ([v] if isinstance(v, str) else [])
                     )

catalog/sites/goodreads.py

@@ -74,14 +74,12 @@ def scrape(self, response=None):
         lang = detect_language(b["title"] + " " + data["brief"])
         data["localized_title"] = [{"lang": lang, "text": b["title"]}]
         data["localized_subtitle"] = []  # Goodreads does not support subtitle
+        if data["brief"]:
+            data["brief"] = html_to_text(data["brief"])
         data["localized_description"] = (
             [{"lang": lang, "text": data["brief"]}] if data["brief"] else []
         )
-
-        if data["brief"]:
-            data["brief"] = re.sub(
-                r"<[^>]*>", "", data["brief"].replace("<br />", "\n")
-            )
+        data["author"] = [c["name"] for c in o["Contributor"] if c.get("name")]
         ids = {}
         t, n = detect_isbn_asin(b["details"].get("asin"))
         if t:
@@ -159,7 +157,7 @@ def scrape(self, response=None):
             metadata={
                 "title": title,
                 "localized_title": [{"lang": "en", "text": title}],
-                "author": author,
+                "author": [author] if author else [],
                 "first_published": first_published,
             }
         )

catalog/templates/_sidebar_edit.html

@@ -157,7 +157,7 @@ <h5>{% trans "Edit Options" %}</h5>
     {% endif %}
     {% if item.class_name == "tvseason" or item.class_name == "performanceproduction" %}
       <details>
-        <summary>{% trans "link to parent item" %}</summary>
+        <summary>{% trans "Link to parent item" %}</summary>
         <form method="post"
               action="{% url 'catalog:assign_parent' item.url_path item.uuid %}"
               onsubmit="return confirm('{% trans "Sure to link?" %}');">
@@ -173,7 +173,7 @@ <h5>{% trans "Edit Options" %}</h5>
     {% endif %}
     {% if item.class_name == "tvshow" %}
       <details>
-        <summary>{% trans "cleanup seasons" %}</summary>
+        <summary>{% trans "Cleanup seasons" %}</summary>
         <form method="post"
               action="{% url 'catalog:remove_unused_seasons' item.url_path item.uuid %}"
               onsubmit="return confirm('{% trans "This operation cannot be undone. Sure to delete?" %}');">
@@ -186,7 +186,7 @@ <h5>{% trans "Edit Options" %}</h5>
     {% endif %}
     {% if not item.journal_exists or request.user.is_staff %}
       <details>
-        <summary>{% trans 'merge' %}</summary>
+        <summary>{% trans 'Merge' %}</summary>
         <form method="post"
               action="{% url 'catalog:merge' item.url_path item.uuid %}">
           {% csrf_token %}
@@ -201,20 +201,13 @@ <h5>{% trans "Edit Options" %}</h5>
                  value="{% trans 'merge to another item' %}">
         </form>
       </details>
-      {% if item.child_items %}
-        <!-- contains child items -->
-      {% elif item.merged_from_items.all %}
-        <!-- merged from other items -->
-      {% else %}
+      {% if item.can_soft_delete %}
         <details>
           <summary>{% trans 'Delete' %}</summary>
           <form method="post"
                 action="{% url 'catalog:delete' item.url_path item.uuid %}">
             {% csrf_token %}
-            <input class="contrast"
-                   type="submit"
-                   {% if item.is_deleted or item.merged_to_item %}disabled{% endif %}
-                   value="{% trans 'delete' %}">
+            <input class="contrast" type="submit" value="{% trans 'delete' %}">
           </form>
         </details>
       {% endif %}

catalog/templates/discover.html

@@ -87,9 +87,9 @@ <h5>{% trans "Collections" %}</h5>
               <li class="card">
                 <a href="{{ item.url }}" title="{{ item.display_title }}">
                   <img src="{{ item.cover|thumb:'normal' }}"
-                       alt="{{ item.display_title }}"
+                       alt="{{ item.title }}"
                        loading="lazy">
-                  <div class="card-title">{{ item.display_title }}</div>
+                  <div class="card-title">{{ item.title }}</div>
                 </a>
               </li>
             {% endfor %}

catalog/views.py

@@ -263,8 +263,13 @@ def discover(request):
         layout = request.user.preference.discover_layout
         identity = request.user.identity
         announcements = []
-        post_ids = cache.get("popular_posts", [])
-        popular_posts = Takahe.get_posts(post_ids).order_by("-published")
+        if settings.DISCOVER_SHOW_POPULAR_POSTS:
+            post_ids = cache.get("popular_posts", [])
+            popular_posts = Takahe.get_posts(post_ids).order_by("-published")
+        else:
+            popular_posts = Takahe.get_public_posts(settings.DISCOVER_SHOW_LOCAL_ONLY)[
+                :20
+            ]
     else:
         identity = None
         layout = []

catalog/views_edit.py

@@ -119,7 +119,9 @@ def edit(request, item_path, item_uuid):
 def delete(request, item_path, item_uuid):
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
     if not request.user.is_staff and item.journal_exists():
-        raise PermissionDenied(_("Insufficient permission"))
+        raise PermissionDenied(_("Item in use."))
+    if not item.can_soft_delete():
+        raise PermissionDenied(_("Item cannot be deleted."))
     if request.POST.get("sure", 0) != "1":
         return render(request, "catalog_delete.html", {"item": item})
     else:

common/templates/404.html

@@ -17,7 +17,7 @@
     <main class="container">
       <article class="error">
         <header>
-          <h3>🤷🏻 {{ exception|default:"" }}</h3>
+          <h3>🤷🏻 {% trans "Something is missing" %}</h3>
         </header>
         {% blocktrans %}You may have visited an incorrect URL, or the content you are looking for has been deleted by the author.{% endblocktrans %}
         <br>

compose.yml

@@ -34,8 +34,10 @@ x-shared:
       NEODB_DISABLE_CRON_JOBS:
       NEODB_SEARCH_PEERS:
       NEODB_MIN_MARKS_FOR_DISCOVER:
-      NEODB_FILTER_LANGUAGE_FOR_DISCOVER:
-      NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER:
+      NEODB_DISCOVER_UPDATE_INTERVAL:
+      NEODB_DISCOVER_FILTER_LANGUAGE:
+      NEODB_DISCOVER_SHOW_LOCAL_ONLY:
+      NEODB_DISCOVER_SHOW_POPULAR_POSTS:
       NEODB_SENTRY_DSN:
       TAKAHE_SENTRY_DSN:
       NEODB_SENTRY_SAMPLE_RATE:

docs/configuration.md

@@ -25,8 +25,9 @@ if you are doing debug or development:
  - `NEODB_SITE_HEAD`
  - `NEODB_SITE_DESCRIPTION`
  - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default. It can includes languages with no UI translations yet, e.g. if set to `ja,en,zh`, NeoDB scraper will fetch catalog metadata in three languages if they are available from third party sites, Japanese users (= whose browser language set to ja-JP) will see English UI with Japanese metadata.
- - `NEODB_FILTER_LANGUAGE_FOR_DISCOVER` - `False` by default; when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`.
- - `NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER` - `False` by default; when set to `True`, only show items marked by local users rather than entire network in `/discover/`
+ - `NEODB_DISCOVER_FILTER_LANGUAGE` - `False` by default; when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`.
+ - `NEODB_DISCOVER_SHOW_LOCAL_ONLY` - `False` by default; when set to `True`, only show items marked by local users rather than entire network on `/discover/`
+ - `NEODB_DISCOVER_UPDATE_INTERVAL` - minutes between each update for popular items on `/discover/`
  - `NEODB_SITE_LINKS` - a list of title and links to show in the footer, comma separated, e.g. `Feedback=https://discord.gg/8KweCuApaK,ToS=/pages/rules/`
  - `NEODB_INVITE_ONLY` - `False` by default, set to `True` to require invite code(generated by `neodb-manage invite --create`) to register
  - `NEODB_ENABLE_LOCAL_ONLY` - `False` by default, set to `True` to allow user to post marks as "local public"

journal/models/mark.py

@@ -250,7 +250,7 @@ def update(
         else:
             # ignore most recent post if exists and create new one
             update_mode = 2
-            shelf = Shelf.objects.get(owner=self.owner, shelf_type=shelf_type)
+            shelf = self.owner.shelf_manager.get_shelf(shelf_type)
             d = {"parent": shelf, "visibility": visibility, "position": 0}
             if metadata:
                 d["metadata"] = metadata