[Backend] : Integrate shinobi into main (#1954)

* add

* pass

* Update get_producer.py

* add

* Maybe

* fix dockerfile

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

backend/Dockerfile

@@ -37,14 +37,15 @@ COPY . /code
 
 # Cleanups
 RUN pip install "pip-autoremove"
-RUN pip-autoremove poetry "pip-autoremove" cython  -y
+RUN pip-autoremove poetry "pip-autoremove" cython 'django-browser-reload'  -y
 RUN apk del nodejs npm
 
 # Remove test folders
 RUN find . -type d -name 'test*' -exec rm -r {} +
 
 RUN chmod +x /code/server_entrypoint.sh
 RUN chmod +x /code/worker_entrypoint.sh
+RUN chmod +x /code/beat_entrypoint.sh
 
 RUN chmod +x /code/entrypoint.sh
 

backend/beat_entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+until cd /code
+do
+    echo "Waiting for server volume..."
+done
+
+# run a worker :)
+export PYTHONPATH=$PYTHONPATH:/code/django_core:/code
+
+celery -A django_core.core beat -l info

backend/django_core/apps/anime/migrations/0010_animemodel_is_locked.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2 on 2023-04-24 07:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("anime", "0009_animegenremodel_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="animemodel",
+            name="is_locked",
+            field=models.BooleanField(default=False),
+            preserve_default=False,
+        ),
+    ]

backend/django_core/apps/anime/models/__init__.py

@@ -1,6 +1,7 @@
 from colorfield.fields import ColorField
 from dynamic_filenames import FilePattern
 from mixins.models.created_at import CreatedAtMixin
+from mixins.models.is_locked import IsLockedMixin
 from mixins.models.updated_at import UpdatedAtMixin
 
 from django.db import models
@@ -29,7 +30,7 @@ class Meta:
         verbose_name = "Anime Synonym"
 
 
-class AnimeModel(CreatedAtMixin, UpdatedAtMixin):
+class AnimeModel(CreatedAtMixin, UpdatedAtMixin, IsLockedMixin):
     mal_id = models.IntegerField(unique=True, blank=True, null=True)
     anilist_id = models.IntegerField(unique=True, blank=True, null=True)
     kitsu_id = models.IntegerField(unique=True, blank=True, null=True)

backend/django_core/apps/anime/tasks.py

@@ -5,9 +5,28 @@
 from colorthief import ColorThief
 from utilities.rgb_to_hex import rgb_to_hex
 
+from shinobi.builder.anime import AnimeBuilder
+
+from django.core.management import call_command
+from django.db.models import Q
+from django.utils import timezone
+
 from .models import AnimeModel
 
 
+@shared_task()
+def get_perodic_animes():
+    builder = AnimeBuilder()
+
+    instances = AnimeModel.objects.filter(
+        Q(updated_at__gte=timezone.now() - timezone.timedelta(days=7)) & Q(is_locked=False)
+    )
+    dictionary = builder.build_dictionary(excluded_ids=instances.values_list(flat=True))
+
+    for anime in list(dictionary.keys()):
+        call_command("get_anime", anime_id=anime)
+
+
 @shared_task()
 def set_field_color(
     pk: int,

backend/django_core/apps/characters/management/commands/get_character.py

@@ -0,0 +1,49 @@
+import sys
+from typing import NoReturn
+
+import httpx
+
+from shinobi.parser.character import CharacterParser
+
+from django.core.management.base import BaseCommand
+
+from ...models import CharacterModel
+
+
+class Command(BaseCommand):
+    help = "Django command that gets the Character Information given mal_id"
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.client = httpx.Client()
+        super().__init__(*args, **kwargs)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "character_number",
+            type=int,
+            help="The character number to get information for",
+        )
+
+    def handle(self, *args, **options) -> NoReturn:
+        character_number: int = options["character_number"]
+        res = self.client.get(f"https://myanimelist.net/character/{character_number}")
+
+        parser = CharacterParser(res.content)
+        data_dictionary = parser.build_dictionary()
+
+        try:
+            character_instance = CharacterModel.objects.get(mal_id=character_number)
+        except CharacterModel.DoesNotExist:
+            self.stdout.write(
+                f"No CharacterModel found for {self.style.ERROR(character_number)}"
+            )
+            sys.exit(1)
+
+        for attr, value in data_dictionary.items():
+            if value:
+                setattr(character_instance, attr, value)
+
+        character_instance.save()
+        self.stdout.write(
+            f"Successfully got info for {self.style.SUCCESS(character_number)}"
+        )

backend/django_core/apps/characters/migrations/0003_charactermodel_is_locked.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2 on 2023-04-24 07:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("characters", "0002_charactermodel_created_at_charactermodel_updated_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="charactermodel",
+            name="is_locked",
+            field=models.BooleanField(default=False),
+            preserve_default=False,
+        ),
+    ]

backend/django_core/apps/characters/models.py

@@ -1,5 +1,6 @@
 from dynamic_filenames import FilePattern
 from mixins.models.created_at import CreatedAtMixin
+from mixins.models.is_locked import IsLockedMixin
 from mixins.models.updated_at import UpdatedAtMixin
 
 from django.db import models
@@ -10,7 +11,7 @@
 # Create your models here.
 
 
-class CharacterModel(CreatedAtMixin, UpdatedAtMixin):
+class CharacterModel(CreatedAtMixin, UpdatedAtMixin, IsLockedMixin):
     mal_id = models.IntegerField(unique=True, null=True)
     kitsu_id = models.IntegerField(unique=True, null=True)
     anilist_id = models.IntegerField(unique=True, null=True)

backend/django_core/apps/characters/tasks.py

@@ -0,0 +1,21 @@
+from celery import shared_task
+from shinobi.builder.character import CharacterBuilder
+
+from django.core.management import call_command
+from django.db.models import Q
+from django.utils import timezone
+
+from .models import CharacterModel
+
+
+@shared_task()
+def get_perodic_character():
+    builder = CharacterBuilder()
+
+    instances = CharacterModel.objects.filter(
+        Q(updated_at__gte=timezone.now() - timezone.timedelta(days=7)) & Q(is_locked=False)
+    )
+    dictionary = builder.build_dictionary(excluded_ids=instances.values_list(flat=True))
+
+    for anime in list(dictionary.keys()):
+        call_command("get_character", character_id=anime)

backend/django_core/apps/producers/management/commands/get_producer.py

@@ -0,0 +1,49 @@
+import sys
+from typing import NoReturn
+
+import httpx
+
+from shinobi.parser.producer import ProducerParser
+
+from django.core.management.base import BaseCommand
+
+from ...models import ProducerModel
+
+
+class Command(BaseCommand):
+    help = "Django command that gets the Producer Information given mal_id"
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.client = httpx.Client()
+        super().__init__(*args, **kwargs)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "producer_number",
+            type=int,
+            help="The producer number to get information for",
+        )
+
+    def handle(self, *args, **options) -> NoReturn:
+        producer_number: int = options["producer_number"]
+        res = self.client.get(f"https://myanimelist.net/anime/producer/{producer_number}")
+
+        parser = ProducerParser(res.content)
+        data_dictionary = parser.build_dictionary()
+
+        try:
+            producer_instance = ProducerModel.objects.get(mal_id=producer_number)
+        except ProducerModel.DoesNotExist:
+            self.stdout.write(
+                f"No ProducerModel found for {self.style.ERROR(producer_number)}"
+            )
+            sys.exit(1)
+
+        for attr, value in data_dictionary.items():
+            if value:
+                setattr(producer_instance, attr, value)
+
+        producer_instance.save()
+        self.stdout.write(
+            f"Successfully got info for {self.style.SUCCESS(producer_number)}"
+        )

backend/django_core/apps/producers/migrations/0005_producermodel_is_locked.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2 on 2023-04-24 07:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("producers", "0004_rename_japanese_title_producermodel_name_japanese"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="producermodel",
+            name="is_locked",
+            field=models.BooleanField(default=False),
+            preserve_default=False,
+        ),
+    ]

backend/django_core/apps/producers/models.py

@@ -1,12 +1,13 @@
 from mixins.models.created_at import CreatedAtMixin
+from mixins.models.is_locked import IsLockedMixin
 from mixins.models.updated_at import UpdatedAtMixin
 
 from django.db import models
 
 # Create your models here.
 
 
-class ProducerModel(CreatedAtMixin, UpdatedAtMixin):
+class ProducerModel(CreatedAtMixin, UpdatedAtMixin, IsLockedMixin):
     mal_id = models.IntegerField(unique=True, blank=True, null=True)
     kitsu_id = models.IntegerField(unique=True, blank=True, null=True)
     name = models.CharField(

backend/django_core/apps/staffs/management/commands/get_staff.py

@@ -0,0 +1,45 @@
+import sys
+from typing import NoReturn
+
+import httpx
+
+from shinobi.parser.staff import StaffParser
+
+from django.core.management.base import BaseCommand
+
+from ...models import StaffModel
+
+
+class Command(BaseCommand):
+    help = "Django command that gets the Staff Information given mal_id"
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.client = httpx.Client()
+        super().__init__(*args, **kwargs)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "staff_number",
+            type=int,
+            help="The staff number to get information for",
+        )
+
+    def handle(self, *args, **options) -> NoReturn:
+        staff_number: int = options["staff_number"]
+        res = self.client.get(f"https://myanimelist.net/people/{staff_number}")
+
+        parser = StaffParser(res.content)
+        data_dictionary = parser.build_dictionary()
+
+        try:
+            staff_instance = StaffModel.objects.get(mal_id=staff_number)
+        except StaffModel.DoesNotExist:
+            self.stdout.write(f"No StaffModel found for {self.style.ERROR(staff_number)}")
+            sys.exit(1)
+
+        for attr, value in data_dictionary.items():
+            if value:
+                setattr(staff_instance, attr, value)
+
+        staff_instance.save()
+        self.stdout.write(f"Successfully got info for {self.style.SUCCESS(staff_number)}")

backend/django_core/apps/staffs/migrations/0004_staffmodel_is_locked.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2 on 2023-04-24 07:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("staffs", "0003_staffmodel_created_at_staffmodel_updated_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="staffmodel",
+            name="is_locked",
+            field=models.BooleanField(default=False),
+            preserve_default=False,
+        ),
+    ]

backend/django_core/apps/staffs/models.py

@@ -1,6 +1,7 @@
 from core.storages import OverwriteStorage
 from dynamic_filenames import FilePattern
 from mixins.models.created_at import CreatedAtMixin
+from mixins.models.is_locked import IsLockedMixin
 from mixins.models.updated_at import UpdatedAtMixin
 
 from django.db import models
@@ -19,7 +20,7 @@ class Meta:
         verbose_name_plural = "Staff | People ( Alternate Names )"
 
 
-class StaffModel(CreatedAtMixin, UpdatedAtMixin):
+class StaffModel(CreatedAtMixin, UpdatedAtMixin, IsLockedMixin):
     mal_id = models.IntegerField(unique=True, null=True, blank=True)
     kitsu_id = models.IntegerField(unique=True, null=True, blank=True)
     anilist_id = models.IntegerField(unique=True, null=True, blank=True)

backend/django_core/core/settings.py

@@ -213,7 +213,7 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.postgresql",
-        "NAME": os.environ.get("POSTGRES_NAME", "django"),
+        "NAME": os.environ.get("POSTGRES_NAME", "postgres"),
         "USER": os.environ.get("POSTGRES_USER", "postgres"),
         "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "supersecretpassword"),
         "HOST": os.environ.get("POSTGRES_HOST", "localhost"),

backend/django_core/mixins/models/is_locked.py

@@ -0,0 +1,8 @@
+from django.db import models
+
+
+class IsLockedMixin(models.Model):
+    is_locked = models.BooleanField()
+
+    class Meta:
+        abstract = True