Task to update Lutris DB with new games using MobyGames API

Makefile

@@ -32,7 +32,7 @@ builddeps:
 
 serverdeps:
 	sudo apt-get update
-	sudo apt-get -y --allow-unauthenticated install nginx supervisor rabbitmq-server locales
+	sudo apt-get -y --allow-unauthenticated install nginx supervisor locales
 	pip3 install -r config/requirements/production.pip --exists-action=s
 
 deps:

accounts/tasks.py

@@ -1,6 +1,5 @@
 import logging
 
-from celery import task
 from django.db import IntegrityError
 
 import games.models
@@ -9,10 +8,12 @@
 from accounts.models import User
 from common.util import slugify
 
+from lutrisweb import celery_app
+
 LOGGER = logging.getLogger()
 
 
-@task
+@celery_app.task
 def sync_steam_library(user_id):
     user = User.objects.get(pk=user_id)
     steamid = user.steamid
@@ -50,6 +51,6 @@ def sync_steam_library(user_id):
             pass
 
 
-@task
+@celery_app.task
 def daily_mod_mail():
     send_daily_mod_mail()

config/requirements/base.pip

@@ -1,7 +1,7 @@
 beautifulsoup4==4.6.0
 PyYAML==3.12
 ipython==6.4.0
-Celery==4.1.0
+Celery==4.4.2
 cssselect==1.0.3
 Django==2.0.5
 django-appconf==1.0.2

docker-compose.yml

@@ -8,6 +8,9 @@ services:
       context: .
       args:
         - USE_SQLITE=${USE_SQLITE}
+    environment:
+      MOBY_API_KEY
+    restart: on-failure
     volumes:
       - .:/app
       - /app/media
@@ -18,13 +21,20 @@ services:
     tty: true
     depends_on:
       - db
+      - rabbitmq
   db:
     image: postgres:11
     ports:
       - 5432:5432
+    restart: on-failure
     volumes:
       - postgresql:/var/lib/postgresql/data
     environment:
       POSTGRES_DB: 'lutris'
       POSTGRES_USER: 'lutris'
       POSTGRES_PASSWORD: 'admin'
+  rabbitmq:
+    image: rabbitmq
+    ports:
+      - 5672:5672
+    restart: on-failure

games/tasks.py

@@ -1,21 +1,24 @@
 """Celery tasks for account related jobs"""
 import logging
-from celery import task
+
+import requests
+from django.conf import settings
 from reversion.models import Version, Revision
 from django.contrib.contenttypes.models import ContentType
 from games import models
+from lutrisweb import celery_app
 
 LOGGER = logging.getLogger(__name__)
 
 
-@task
+@celery_app.task
 def delete_unchanged_forks():
     """Periodically delete forked installers that haven't received any changes"""
     for installer in models.Installer.objects.abandoned():
         installer.delete()
 
 
-@task
+@celery_app.task
 def clear_orphan_versions():
     """Deletes versions that are no longer associated with an installer"""
     content_type = ContentType.objects.get_for_model(models.Installer)
@@ -26,7 +29,42 @@ def clear_orphan_versions():
         version.delete()
 
 
-@task
+@celery_app.task
 def clear_orphan_revisions():
     """Clear revisions that are no longer attached to any object"""
     Revision.objects.filter(version__isnull=True).delete()
+
+
+@celery_app.task
+def add_new_games():
+    """Check MobyGames for newly added games every 3 hours and update Lutris DB"""
+    #
+    response = requests.get(url="https://api.mobygames.com/v1/games/recent",
+                            params={
+                                'api_key': settings.MOBY_API_KEY,
+                                'format': 'normal',
+                                'age': 1
+                            })
+    if response.status_code == 200:
+        new_entries = 0
+        data = response.json()
+        supported_platforms = models.Platform.objects.values_list('name', flat=True)
+        supported_genres = models.Genre.objects.values_list('name', flat=True)
+        for game in data.get('games', []):
+            game_platforms = [platform.get('platform_name', '') for platform in game.get('platforms', [])
+                              if platform.get('platform_name', '') in supported_platforms]
+            game_genres = [genre.get('genre_name', '') for genre in game.get('genres', [])
+                           if genre.get('genre_name', '') in supported_genres]
+            if game_platforms:
+                obj, created = models.Game.objects.get_or_create(
+                    name=game.get('title', ''),
+                    defaults={
+                        'description': game.get('description', '')
+                    }
+                )
+                if created:
+                    new_entries += 1
+                    obj.platforms.add(*models.Platform.objects.filter(name__in=game_platforms))
+                    obj.genres.add(*models.Genre.objects.filter(name__in=game_genres))
+    else:
+        pass

lutrisweb/__init__.py

@@ -1,3 +1,7 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
-from .celery import app  # noqa
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app  as celery_app # noqa
+
+__all__ = ('celery_app',)

lutrisweb/celery.py

@@ -9,8 +9,8 @@
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lutrisweb.settings.local')
 
 app = Celery('lutrisweb')
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
 
 
 @app.task(bind=True)

lutrisweb/settings/base.py

@@ -82,6 +82,7 @@ def media_directory(path):
 TGD_LUTRIS_BANNER_PATH = media_directory('thegamesdb/lutris-banners')
 
 SECRET_KEY = os.environ.get('SECRET_KEY', 'changeme')
+MOBY_API_KEY = os.environ.get('MOBY_API_KEY', 'Provide API key')
 
 MIDDLEWARE = [
     # Caching disabled until proper invalidation is implemented
@@ -228,6 +229,10 @@ def media_directory(path):
     'clear-orphan-revisions': {
         'task': 'games.tasks.clear_orphan_revisions',
         'schedule': crontab(hour=17, minute=30)
+    },
+    'add-new-games': {
+        'task': 'games.tasks.add_new_games',
+        'schedule': crontab(minute=0, hour='*/3')
     }
 }
 BROKER_URL = 'amqp://guest:guest@localhost//'