From d38325c5a6a08fba7ab5bb1430ae0fd9160b0de0 Mon Sep 17 00:00:00 2001 From: Her Email Date: Wed, 8 Nov 2023 09:54:36 -0500 Subject: [PATCH] migrate and sync user relationship in Mastodon --- common/management/commands/cron.py | 1 + neodb-takahe | 2 +- users/data.py | 19 +++++-- users/jobs/__init__.py | 1 + users/jobs/sync.py | 35 +++++++++++++ users/management/commands/refresh_mastodon.py | 10 ---- ...nce_mastodon_skip_relationship_and_more.py | 44 ++++++++++++++++ users/models/apidentity.py | 2 +- users/models/preference.py | 2 + users/models/user.py | 50 ++++++++++++++++--- users/tasks.py | 24 --------- users/templates/users/account.html | 39 +++++++++++++-- users/urls.py | 5 ++ 13 files changed, 181 insertions(+), 53 deletions(-) create mode 100644 users/jobs/__init__.py create mode 100644 users/jobs/sync.py delete mode 100644 users/management/commands/refresh_mastodon.py create mode 100644 users/migrations/0014_preference_mastodon_skip_relationship_and_more.py diff --git a/common/management/commands/cron.py b/common/management/commands/cron.py index a4dd9e4e..2e093f35 100644 --- a/common/management/commands/cron.py +++ b/common/management/commands/cron.py @@ -3,6 +3,7 @@ from catalog.jobs import * # noqa from common.models import JobManager +from users.jobs import * # noqa class Command(BaseCommand): diff --git a/neodb-takahe b/neodb-takahe index 25ead63f..401be8d9 160000 --- a/neodb-takahe +++ b/neodb-takahe @@ -1 +1 @@ -Subproject commit 25ead63fb1cacb34cf3f8a2e0706843636f78034 +Subproject commit 401be8d99e793f05761802ea7ebaae8449a44974 diff --git a/users/data.py b/users/data.py index eea7213c..c6bcdd3e 100644 --- a/users/data.py +++ b/users/data.py @@ -13,7 +13,6 @@ from journal.importers.goodreads import GoodreadsImporter from journal.importers.opml import OPMLImporter from journal.models import reset_journal_visibility_for_user -from mastodon import mastodon_request_included from mastodon.api import * from social.models import reset_social_visibility_for_user @@ -21,7 +20,6 @@ from .tasks import * -@mastodon_request_included @login_required def preferences(request): preference = request.user.preference @@ -54,7 +52,6 @@ def preferences(request): return render(request, "users/preferences.html") -@mastodon_request_included @login_required def data(request): return render( @@ -79,7 +76,6 @@ def data_import_status(request): ) -@mastodon_request_included @login_required def export_reviews(request): if request.method != "POST": @@ -87,7 +83,6 @@ def export_reviews(request): return render(request, "users/data.html") -@mastodon_request_included @login_required def export_marks(request): if request.method == "POST": @@ -120,6 +115,20 @@ def sync_mastodon(request): return redirect(reverse("users:info")) +@login_required +def sync_mastodon_preference(request): + if request.method == "POST": + request.user.preference.mastodon_skip_userinfo = ( + request.POST.get("mastodon_sync_userinfo", "") == "" + ) + request.user.preference.mastodon_skip_relationship = ( + request.POST.get("mastodon_sync_relationship", "") == "" + ) + request.user.preference.save() + messages.add_message(request, messages.INFO, _("同步设置已保存。")) + return redirect(reverse("users:info")) + + @login_required def reset_visibility(request): if request.method == "POST": diff --git a/users/jobs/__init__.py b/users/jobs/__init__.py new file mode 100644 index 00000000..e88820ee --- /dev/null +++ b/users/jobs/__init__.py @@ -0,0 +1 @@ +from .sync import MastodonUserSync diff --git a/users/jobs/sync.py b/users/jobs/sync.py new file mode 100644 index 00000000..6b7d89cd --- /dev/null +++ b/users/jobs/sync.py @@ -0,0 +1,35 @@ +import pprint +from datetime import timedelta +from time import sleep + +from django.utils import timezone +from loguru import logger + +from common.models import BaseJob, JobManager +from users.models import Preference, User + + +@JobManager.register +class MastodonUserSync(BaseJob): + interval = timedelta(hours=2) + + def run(self): + logger.info("Mastodon User Sync start.") + count = 0 + ttl_hours = 12 + qs = ( + User.objects.exclude( + preference__mastodon_skip_userinfo=True, mastodon_skip_relationship=True + ) + .filter( + mastodon_last_refresh__lt=timezone.now() - timedelta(hours=ttl_hours) + ) + .filter( + is_active=True, + ) + .exclude(mastodon_token__isnull=True) + .exclude(mastodon_token="") + ) + for user in qs: + user.refresh_mastodon_data() + logger.info(f"Mastodon User Sync finished.") diff --git a/users/management/commands/refresh_mastodon.py b/users/management/commands/refresh_mastodon.py deleted file mode 100644 index 8d6ee755..00000000 --- a/users/management/commands/refresh_mastodon.py +++ /dev/null @@ -1,10 +0,0 @@ -from django.core.management.base import BaseCommand - -from users.tasks import refresh_all_mastodon_data_task - - -class Command(BaseCommand): - help = "Refresh Mastodon data for all users if not updated in last 24h" - - def handle(self, *args, **options): - refresh_all_mastodon_data_task(24) diff --git a/users/migrations/0014_preference_mastodon_skip_relationship_and_more.py b/users/migrations/0014_preference_mastodon_skip_relationship_and_more.py new file mode 100644 index 00000000..7ea9608d --- /dev/null +++ b/users/migrations/0014_preference_mastodon_skip_relationship_and_more.py @@ -0,0 +1,44 @@ +# Generated by Django 4.2.7 on 2023-11-06 01:46 + +from django.db import migrations, models +from loguru import logger +from tqdm import tqdm + + +def migrate_relationships(apps, schema_editor): + User = apps.get_model("users", "User") + APIdentity = apps.get_model("users", "APIdentity") + logger.info(f"Migrate user relationship") + for user in tqdm(User.objects.all()): + for target in user.local_following: + user.identity.follow(User.objects.get(pk=target).identity) + for target in user.local_blocking: + user.identity.block(User.objects.get(pk=target).identity) + for target in user.local_muting: + user.identity.block(User.objects.get(pk=target).identity) + user.sync_relationship() + for user in tqdm(User.objects.all()): + for req in user.identity.following_request: + target_identity = APIdentity.objects.get(pk=req) + target_identity.accept_follow_request(user.identity) + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0013_init_identity"), + ] + + operations = [ + migrations.RunPython(migrate_relationships), + migrations.AddField( + model_name="preference", + name="mastodon_skip_relationship", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="preference", + name="mastodon_skip_userinfo", + field=models.BooleanField(default=False), + ), + ] diff --git a/users/models/apidentity.py b/users/models/apidentity.py index 7d61bfc1..433fbfea 100644 --- a/users/models/apidentity.py +++ b/users/models/apidentity.py @@ -82,7 +82,7 @@ def avatar(self): return ( self.takahe_identity.icon.url if self.takahe_identity.icon - else settings.SITE_INFO["user_icon"] + else self.takahe_identity.icon_uri or settings.SITE_INFO["user_icon"] ) else: return f"/proxy/identity_icon/{self.pk}/" diff --git a/users/models/preference.py b/users/models/preference.py index ac5a0b45..691d0220 100644 --- a/users/models/preference.py +++ b/users/models/preference.py @@ -49,6 +49,8 @@ class Preference(models.Model): show_last_edit = models.PositiveSmallIntegerField(default=0) no_anonymous_view = models.PositiveSmallIntegerField(default=0) hidden_categories = models.JSONField(default=list) + mastodon_skip_userinfo = models.BooleanField(null=False, default=False) + mastodon_skip_relationship = models.BooleanField(null=False, default=False) def __str__(self): return str(self.user) diff --git a/users/models/user.py b/users/models/user.py index 7cc4dc0c..5533ed78 100644 --- a/users/models/user.py +++ b/users/models/user.py @@ -231,12 +231,44 @@ def clear(self): self.identity.deleted = timezone.now() self.identity.save() - def sync_relationships(self): - # FIXME - pass + def sync_relationship(self): + for target in self.mastodon_followers: + t = target.split("@") + target_user = User.objects.filter( + mastodon_username=t[0], mastodon_site=t[1] + ).first() + if target_user and not self.identity.is_following(target_user.identity): + self.identity.follow(target_user.identity) + for target in self.mastodon_blocks: + t = target.split("@") + target_user = User.objects.filter( + mastodon_username=t[0], mastodon_site=t[1] + ).first() + if target_user and not self.identity.is_blocking(target_user.identity): + self.identity.block(target_user.identity) + for target in self.mastodon_mutes: + t = target.split("@") + target_user = User.objects.filter( + mastodon_username=t[0], mastodon_site=t[1] + ).first() + if target_user and not self.identity.is_muting(target_user.identity): + self.identity.mute(target_user.identity) + + def sync_identity(self): + identity = self.identity.takahe_identity + identity.name = ( + self.mastodon_account.get("display_name") + or identity.name + or identity.username + ) + identity.summary = self.mastodon_account.get("note") or identity.summary + identity.manually_approves_followers = self.mastodon_locked + identity.icon_uri = self.mastodon_account.get("avatar") + identity.save() def refresh_mastodon_data(self): """Try refresh account data from mastodon server, return true if refreshed successfully, note it will not save to db""" + logger.debug(f"Refreshing Mastodon data for {self}") self.mastodon_last_refresh = timezone.now() code, mastodon_account = verify_account(self.mastodon_site, self.mastodon_token) if code == 401 and self.mastodon_refresh_token: @@ -247,7 +279,6 @@ def refresh_mastodon_data(self): code, mastodon_account = verify_account( self.mastodon_site, self.mastodon_token ) - updated = False if mastodon_account: self.mastodon_account = mastodon_account self.mastodon_locked = mastodon_account["locked"] @@ -277,12 +308,17 @@ def refresh_mastodon_data(self): self.mastodon_domain_blocks = get_related_acct_list( self.mastodon_site, self.mastodon_token, "/api/v1/domain_blocks" ) - self.sync_relationships() - updated = True + self.save() + if not self.preference.mastodon_skip_userinfo: + self.sync_identity() + if not self.preference.mastodon_skip_relationship: + self.sync_relationship() + return True elif code == 401: logger.error(f"Refresh mastodon data error 401 for {self}") self.mastodon_token = "" - return updated + self.save(update_fields=["mastodon_token"]) + return False @property def unread_announcements(self): diff --git a/users/tasks.py b/users/tasks.py index c6cb4c8a..18ac042d 100644 --- a/users/tasks.py +++ b/users/tasks.py @@ -16,30 +16,6 @@ def refresh_mastodon_data_task(user_id, token=None): if token: user.mastodon_token = token if user.refresh_mastodon_data(): - user.save() logger.info(f"{user} mastodon data refreshed") else: logger.warning(f"{user} mastodon data refresh failed") - - -def refresh_all_mastodon_data_task(ttl_hours): - logger.info(f"Mastodon data refresh start") - count = 0 - for user in tqdm( - User.objects.filter( - mastodon_last_refresh__lt=timezone.now() - timedelta(hours=ttl_hours), - is_active=True, - ) - ): - if user.mastodon_token or user.mastodon_refresh_token: - logger.info(f"Refreshing {user}") - if user.refresh_mastodon_data(): - logger.info(f"Refreshed {user}") - count += 1 - else: - logger.warning(f"Refresh failed for {user}") - user.save() - else: - logger.warning(f"Missing token for {user}") - logger.info(f"{count} users updated") - logger.info(f"Mastodon data refresh done") diff --git a/users/templates/users/account.html b/users/templates/users/account.html index a960bb47..659bc382 100644 --- a/users/templates/users/account.html +++ b/users/templates/users/account.html @@ -139,20 +139,49 @@
- {% trans '同步联邦宇宙身份和社交关系数据' %} + {% trans '同步联邦宇宙信息和社交数据' %} +
+ {% csrf_token %} +
+ +
+
+ +
+ + + {{ site_name }}会按照以上设置每天自动导入你在联邦宇宙实例中新增的关注、屏蔽和隐藏列表； +
+ 如果你在联邦宇宙实例中关注的用户加入了NeoDB，你会自动关注她； +
+ 如果你在联邦宇宙实例中取消了关注、屏蔽或隐藏，{{ site_name }}不会自动取消，但你可以手动移除。 + +
{% csrf_token %} + 如果希望立即开始同步，可以点击下方按钮。 {% if request.user.mastodon_last_refresh %}上次更新时间 {{ request.user.mastodon_last_refresh }}{% endif %} -
- 为了正确高效的展示短评和评论，{{ site_name }}会缓存你在联邦宇宙的关注、屏蔽和隐藏列表。如果你刚刚更新过帐户的上锁状态、增减过关注、隐藏或屏蔽，希望立即生效，可以点击这里立刻更新；这类信息也会每天自动同步。 -
diff --git a/users/urls.py b/users/urls.py index f1ca1753..0f9c5026 100644 --- a/users/urls.py +++ b/users/urls.py @@ -23,6 +23,11 @@ path("data/export/reviews", export_reviews, name="export_reviews"), path("data/export/marks", export_marks, name="export_marks"), path("data/sync_mastodon", sync_mastodon, name="sync_mastodon"), + path( + "data/sync_mastodon_preference", + sync_mastodon_preference, + name="sync_mastodon_preference", + ), path("data/reset_visibility", reset_visibility, name="reset_visibility"), path("data/clear_data", clear_data, name="clear_data"), path("preferences", preferences, name="preferences"),