Merge branch 'master' into docker

Procfile

@@ -1,2 +1,3 @@
-web: gunicorn mygpo.wsgi:application -c gunicorn.conf.py
-beat: python manage.py celery beat -S django --pidfile /var/run/mygpo/celerybeat.pid
+web: gunicorn mygpo.wsgi:application -c conf/gunicorn.conf.py
+beat: celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django
+celery: celery -A mygpo worker --concurrency=3 -l info -Ofair

doc/dev/installation.rst

@@ -43,20 +43,7 @@ Now install additional dependencies locally:
     pip install -r requirements-test.txt   # for running tests
 
 
-That's it for the setup. Now to initialize the DB:
-
-First run the commands from :ref:`db-setup`. Then
-
-.. code-block:: bash
-
-    cd mygpo
-    python manage.py migrate
-
-..and here we go:
-
-.. code-block:: bash
-
-    python manage.py runserver
+That's it for the setup.
 
 
 Configuration
@@ -75,6 +62,26 @@ For a development configuration you will probably want to use the following
 See :ref:`configuration` for further information.
 
 
+Database Initialization
+-----------------------
+
+Now to initialize the DB:
+
+First run the commands from :ref:`db-setup`. Then
+
+.. code-block:: bash
+
+    cd mygpo
+    envdir envs/local python manage.py migrate
+
+..and here we go:
+
+.. code-block:: bash
+
+    envdir envs/local python manage.py runserver
+
+
+
 Accessing the dev server from other devices
 -------------------------------------------
 
@@ -84,7 +91,7 @@ runserver command of manage.py, like this:
 
 .. code-block:: bash
 
-    python manage.py runserver 0.0.0.0:8000
+    envdir envs/local python manage.py runserver 0.0.0.0:8000
 
 Beware, though, that this will expose the web service to your all networks
 that your machine is connected to. Apply common sense and ideally use only
@@ -101,22 +108,22 @@ commands regularly on your development machine:
 
 .. code-block:: bash
 
-    python manage.py update-categories
-    python manage.py update-toplist
-    python manage.py update-episode-toplist
+    envdir envs/local python manage.py update-categories
+    envdir envs/local python manage.py update-toplist
+    envdir envs/local python manage.py update-episode-toplist
 
-    python manage.py feed-downloader
-    python manage.py feed-downloader <feed-url> [...]
-    python manage.py feed-downloader --max <max-updates>
-    python manage.py feed-downloader --random --max <max-updates>
-    python manage.py feed-downloader --toplist --max <max-updates>
-    python manage.py feed-downloader --update-new --max <max-updates>
+    envdir envs/local python manage.py feed-downloader
+    envdir envs/local python manage.py feed-downloader <feed-url> [...]
+    envdir envs/local python manage.py feed-downloader --max <max-updates>
+    envdir envs/local python manage.py feed-downloader --random --max <max-updates>
+    envdir envs/local python manage.py feed-downloader --toplist --max <max-updates>
+    envdir envs/local python manage.py feed-downloader --update-new --max <max-updates>
 
 or to only do a dry run (this won't do any web requests for feeds):
 
 .. code-block:: bash
 
-    python manage.py feed-downloader --list-only [other parameters]
+    envdir envs/local python manage.py feed-downloader --list-only [other parameters]
 
 
 Maintaining publisher relationships with user accounts
@@ -127,7 +134,7 @@ To set a user as publisher for a given feed URL, use:
 .. code-block:: bash
 
     cd mygpo
-    python manage.py make-publisher <username> <feed-url> [...]
+    envdir envs/local python manage.py make-publisher <username> <feed-url> [...]
 
 
 Web-Server
@@ -138,7 +145,7 @@ directory with
 
 .. code-block:: bash
 
-    python manage.py runserver
+    envdir envs/local python manage.py runserver
 
 If you want to run a production server, check out `Deploying Django
 <https://docs.djangoproject.com/en/dev/howto/deployment/>`_.

htdocs/media/screen.css

@@ -1,3 +1,9 @@
+:root {
+    --color-status-success: 90;
+    --color-status-error: 0;
+    --color-status-neutral: 247;
+}
+
 
 /* Landscape phones and down */
 @media (min-width: 980px)
@@ -628,3 +634,16 @@ div.podcasts div.podcast:hover div.actions button.btn
 .hosting {
     text-align: center;
 }
+
+.status-success {
+    background-color: hsla(var(--color-status-success), 100%, 75%, 1);
+}
+
+
+.status-error {
+    background-color: hsla(var(--color-status-error), 100%, 75%, 1);
+}
+
+.status-neutral {
+    background-color: hsla(var(--color-status-neutral), 16%, 85%, 1);
+}

mygpo/administration/templates/admin/hostinfo.html

@@ -1,5 +1,6 @@
 {% extends "base.html" %}
 {% load i18n %}
+{% load time %}
 {% load podcasts %}
 
 {% load menu %}
@@ -62,6 +63,15 @@ <h1>{% trans "Host Information" %}</h1>
    <td class="numeric">{{ num_index_outdated }}</td>
   </tr>
 
+  <tr>
+   <td>
+    <strong>
+     {% trans "Average podcast update duration" %}
+    </strong>
+   </td>
+   <td class="numeric">{{ avg_podcast_update_duration.total_seconds|format_duration}}</td>
+  </tr>
+
   <tr>
    <td><strong>{% trans "Scheduled Celery Tasks" %}</strong></td>
    <td><tt>{{ num_celery_tasks }}</tt></td>

mygpo/administration/tests.py

@@ -24,22 +24,22 @@ class SimpleTest(TestCase):
 
     def test_merge(self):
 
-        p1 = Podcast.objects.get_or_create_for_url('http://example.com/podcast1.rss')
-        p2 = Podcast.objects.get_or_create_for_url('http://example.com/podcast2.rss')
+        p1 = Podcast.objects.get_or_create_for_url('http://example.com/podcast1.rss').object
+        p2 = Podcast.objects.get_or_create_for_url('http://example.com/podcast2.rss').object
 
-        e1 = Episode.objects.get_or_create_for_url(p1, 'http://example.com/podcast1/e1.mp3')
+        e1 = Episode.objects.get_or_create_for_url(p1, 'http://example.com/podcast1/e1.mp3').object
         e1.title = 'Episode 1'
         e1.save()
 
-        e2 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast1/e2.mp3')
+        e2 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast1/e2.mp3').object
         e2.title = 'Episode 2'
         e2.save()
 
-        e3 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast2/e2.mp3')
+        e3 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast2/e2.mp3').object
         e3.title = 'Episode 3'
         e3.save()
 
-        e4 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast2/e3.mp3')
+        e4 = Episode.objects.get_or_create_for_url(p2, 'http://example.com/podcast2/e3.mp3').object
         e4.title = 'Episode 4'
         e4.save()
 

mygpo/administration/views.py

@@ -4,7 +4,10 @@
 from collections import Counter
 from datetime import datetime
 
+import redis
+
 import django
+from django.db.models import Avg
 from django.shortcuts import render
 from django.contrib import messages
 from django.urls import reverse
@@ -26,6 +29,7 @@
 from mygpo.administration.clients import UserAgentStats, ClientStats
 from mygpo.administration.tasks import merge_podcasts
 from mygpo.utils import get_git_head
+from mygpo.data.models import PodcastUpdateResult
 from mygpo.users.models import UserProxy
 from mygpo.publisher.models import PublishedPodcast
 from mygpo.api.httpresponse import JsonResponse
@@ -57,27 +61,36 @@ def get(self, request):
         hostname = socket.gethostname()
         django_version = django.VERSION
 
-        i = celery.control.inspect()
-        scheduled = i.scheduled()
-        if not scheduled:
-            num_celery_tasks = None
-        else:
-            num_celery_tasks = sum(len(node) for node in scheduled.values())
-
         feed_queue_status = self._get_feed_queue_status()
         num_index_outdated = self._get_num_outdated_search_index()
+        avg_podcast_update_duration = self._get_avg_podcast_update_duration()
 
         return self.render_to_response({
             'git_commit': commit,
             'git_msg': msg,
             'base_dir': base_dir,
             'hostname': hostname,
             'django_version': django_version,
-            'num_celery_tasks': num_celery_tasks,
+            'num_celery_tasks': self._get_waiting_celery_tasks(),
+            'avg_podcast_update_duration': avg_podcast_update_duration,
             'feed_queue_status': feed_queue_status,
             'num_index_outdated': num_index_outdated,
         })
 
+    def _get_waiting_celery_tasks(self):
+        con = celery.broker_connection()
+
+        args = {'host': con.hostname}
+        if con.port:
+            args['port'] = con.port
+
+        r = redis.StrictRedis(**args)
+        return r.llen('celery')
+
+    def _get_avg_podcast_update_duration(self):
+        queryset = PodcastUpdateResult.objects.filter(successful=True)
+        return queryset.aggregate(avg_duration=Avg('duration'))['avg_duration']
+
     def _get_feed_queue_status(self):
         now = datetime.utcnow()
         next_podcast = Podcast.objects.all().order_by_next_update().first()

mygpo/api/advanced/__init__.py

@@ -200,8 +200,8 @@ def update_episodes(user, actions, now, ua_string):
         if not episode_url:
             continue
 
-        podcast = Podcast.objects.get_or_create_for_url(podcast_url)
-        episode = Episode.objects.get_or_create_for_url(podcast, episode_url)
+        podcast = Podcast.objects.get_or_create_for_url(podcast_url).object
+        episode = Episode.objects.get_or_create_for_url(podcast, episode_url).object
 
         # parse_episode_action returns a EpisodeHistoryEntry obj
         history = parse_episode_action(action, user, update_urls, now,

mygpo/api/advanced/episode.py

@@ -54,8 +54,8 @@ def get(self, request, username):
 
     def update_chapters(self, req, user):
         """ Add / remove chapters according to the client's request """
-        podcast = Podcast.objects.get_or_create_for_url(podcast_url)
-        episode = Episode.objects.get_or_create_for_url(podcast, episode_url)
+        podcast = Podcast.objects.get_or_create_for_url(podcast_url).object
+        episode = Episode.objects.get_or_create_for_url(podcast, episode_url).object
 
         # add chapters
         for chapter_data in req.get('chapters_add', []):

mygpo/api/advanced/lists.py

@@ -61,7 +61,7 @@ def create(request, username, format):
         return HttpResponse('List already exists', status=409)
 
     urls = parse_subscription(request.body.decode('utf-8'), format)
-    podcasts = [Podcast.objects.get_or_create_for_url(url) for url in urls]
+    podcasts = [Podcast.objects.get_or_create_for_url(url).object for url in urls]
 
     for podcast in podcasts:
         plist.add_entry(podcast)
@@ -148,7 +148,7 @@ def get_list(request, plist, owner, format):
 def update_list(request, plist, owner, format):
     """ Replaces the podcasts in the list and returns 204 No Content """
     urls = parse_subscription(request.body.decode('utf-8'), format)
-    podcasts = [Podcast.objects.get_or_create_for_url(url) for url in urls]
+    podcasts = [Podcast.objects.get_or_create_for_url(url).object for url in urls]
     plist.set_entries(podcasts)
 
     return HttpResponse(status=204)

mygpo/api/legacy.py

@@ -55,11 +55,11 @@ def upload(request):
     rem = list(set(rem))
 
     for n in new:
-        p = Podcast.objects.get_or_create_for_url(n)
+        p = Podcast.objects.get_or_create_for_url(n).object
         subscribe(p, user, dev)
 
     for r in rem:
-        p = Podcast.objects.get_or_create_for_url(r)
+        p = Podcast.objects.get_or_create_for_url(r).object
         unsubscribe(p, user, dev)
 
     return HttpResponse('@SUCCESS', content_type='text/plain')

mygpo/api/simple.py

@@ -200,7 +200,7 @@ def set_subscriptions(urls, user, device_uid, user_agent):
         unsubscribe(podcast, user, device)
 
     for url in new:
-        podcast = Podcast.objects.get_or_create_for_url(url)
+        podcast = Podcast.objects.get_or_create_for_url(url).object
         subscribe(podcast, user, device, url)
 
     # Only an empty response is a successful response

mygpo/api/subscriptions.py

@@ -81,7 +81,7 @@ def update_subscriptions(self, user, device, add, remove):
         rem_s = filter(lambda x: x not in add_s, rem_s)
 
         for add_url in add_s:
-            podcast = Podcast.objects.get_or_create_for_url(add_url)
+            podcast = Podcast.objects.get_or_create_for_url(add_url).object
             subscribe(podcast, user, device, add_url)
 
         remove_podcasts = Podcast.objects.filter(urls__url__in=rem_s)

mygpo/api/tests.py

@@ -168,14 +168,14 @@ def setUp(self):
             defaults = {
                 'title': 'My Podcast',
             },
-        )
+        ).object
         self.episode = Episode.objects.get_or_create_for_url(
             self.podcast,
             'http://example.com/directory-podcast/1.mp3',
             defaults = {
                 'title': 'My Episode',
             },
-        )
+        ).object
         self.client = Client()
 
     def test_episode_info(self):

mygpo/core/slugs.py

@@ -7,10 +7,6 @@ class SlugGenerator(object):
     """ Generates a unique slug for an object """
 
     def __init__(self, obj):
-        if obj.slug:
-            raise ValueError('%(obj)s already has slug %(slug)s' %
-                             dict(obj=obj, slug=obj.slug))
-
         self.obj = obj
         self.base_slug = self._get_base_slug(obj)
 
@@ -26,6 +22,10 @@ def __iter__(self):
 
         The consumer can can consume until it get's an unused one """
 
+        if obj.slug:
+            # The object already has a slug
+            raise StopIteration
+
         if not self.base_slug:
             raise StopIteration
 

mygpo/data/admin.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.PodcastUpdateResult)
+class PodcastUpdateResultAdmin(admin.ModelAdmin):
+    model = models.PodcastUpdateResult
+
+    list_display = ['title', 'start', 'duration', 'successful',
+                    'episodes_added']
+
+    readonly_fields = ['id', 'podcast_url', 'podcast', 'start', 'duration',
+                       'successful', 'error_message', 'podcast_created',
+                       'episodes_added']
+
+    def title(self, res):
+        return res.podcast or res.podcast_url