Added a Celery task for getting a CSV export of all current tasks

.gitignore

@@ -1,4 +1,3 @@
-tododjango
-todo_django/directtodo
+tododjango.db
+/static/
 *.pyc
-*.swp

README.md

@@ -1,7 +1,7 @@
-#Todo-Django
+# Todo-Django
 A django web app for basic task lists that are segmented by user.
 
-##Setup Instructions 
+## Setup Instructions
 Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2.6+ should work.
 - *It is recommended you use a virtual environment*
 - $ pip install -r requirements.txt
@@ -22,7 +22,7 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2.
 - Single UI for most important tasks
   - creation, search, and deleting of tasks is all AJAX
 
-###Includes many features inherited from Django:
+### Includes many features inherited from Django:
   - Robust Admin
   - Supports multiple databases
   - Robust templating engine
@@ -31,6 +31,33 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2.
 - [DirectEmployers UI Framework](https://github.com/DirectEmployers/UI-Framework) (based on Bootstrap)
 - [Django-Taggit](https://github.com/alex/django-taggit)
 
+## Enhancements
+
+### Celery CSV report
+
+You now have the ability to download a CSV of all of your tasks!
+The CSV is generated in the background using
+[Celery](http://www.celeryproject.org/).
+We're also using [Jobtastic](http://policystat.github.io/jobtastic/)
+on the python side to add easy progress reporting and caching.
+On the client side, we're using
+[jquery-celery](https://github.com/PolicyStat/jquery-celery)
+to provide a Bootstrap progress bar for friendly reporting of the CSV progress.
+
+#### Installation
+
+Install [Celery](http://www.celeryproject.org/)
+and [Jobtastic](http://policystat.github.io/jobtastic/),
+then build the database tables required by Django-Celery.
+
+    $ pip install -r requirements.txt
+    $ python manage.py syncdb
+
+You'll also need to run a celery worker.
+
+    $ python manage.py celery worker
+
+
 -----------------------------
 # IndyPy Web Framework Shootout
 

deploy/django.wsgi

@@ -10,5 +10,9 @@ from django.conf import settings
 os.environ["DJANGO_SETTINGS_MODULE"] = "todo_django.settings"
 
 from django.core.handlers.wsgi import WSGIHandler
+
+import djcelery
+djcelery.setup_loader()
+
 application = WSGIHandler()
 

requirements.txt

@@ -1,2 +1,4 @@
 django==1.4.3
 django-taggit==0.9.3
+django-celery==3.0.17
+jobtastic==0.2.1

todo_django/settings.py

@@ -1,6 +1,9 @@
 # Django settings for todo_django project.
+import os
 
-DEBUG = True
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+DEBUG = os.environ.get('DJANGO_DEBUG', False)
 TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
@@ -11,12 +14,12 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-        'NAME': 'tododjango',                      # Or path to database file if using sqlite3.
-        'USER': '',                      # Not used with sqlite3.
-        'PASSWORD': '',                  # Not used with sqlite3.
-        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
-        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(PROJECT_ROOT, 'tododjango.db'),
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': '',
     }
 }
 
@@ -56,25 +59,22 @@
 # Don't put anything in this directory yourself; store your static files
 # in apps' "static/" subdirectories and in STATICFILES_DIRS.
 # Example: "/home/media/media.lawrence.com/static/"
-STATIC_ROOT = ''
+STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')
 
 # URL prefix for static files.
 # Example: "http://media.lawrence.com/static/"
 STATIC_URL = '/static/'
 
 # Additional locations of static files
 STATICFILES_DIRS = (
-    # Put strings here, like "/home/html/static" or "C:/www/django/static".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
+    os.path.join(PROJECT_ROOT, 'todo_django', 'static'),
 )
 
 # List of finder classes that know how to find static files in
 # various locations.
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
-#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
 )
 
 # Make this unique, and don't share it with anybody.
@@ -84,7 +84,6 @@
 TEMPLATE_LOADERS = (
     'django.template.loaders.filesystem.Loader',
     'django.template.loaders.app_directories.Loader',
-#     'django.template.loaders.eggs.Loader',
 )
 
 MIDDLEWARE_CLASSES = (
@@ -103,9 +102,6 @@
 WSGI_APPLICATION = 'todo_django.wsgi.application'
 
 TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
 )
 
 INSTALLED_APPS = (
@@ -119,6 +115,8 @@
     'django.contrib.admin',
     'todo_django',
     'taggit',
+    'djcelery',
+    'kombu.transport.django',
 )
 
 # A sample logging configuration. The only tangible logging
@@ -149,3 +147,21 @@
         },
     }
 }
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/tmp/todo_django_cache',
+    },
+}
+
+# Use the database as Celery's broker. This is a horrible production setting,
+# but makes for an easy-to-use demo
+BROKER_URL = 'django://'
+
+import djcelery
+djcelery.setup_loader()
+
+# For progress demo purposes, do we want to take several seconds between each
+# task for the export?
+TODO_EXPORT_VERY_SLOWLY = True

todo_django/static/js/jquery-celery/celery.js

@@ -0,0 +1,74 @@
+jQuery.fn.djcelery = function(options) {
+    var interval_id;
+    var el = this;
+    var url = '/task/' + options.task_id + '/status/';
+    var number_of_errors = 0;
+    if (options.task_id.length !== 36) {
+        url = '';
+    }
+
+    if (typeof(options.on_failure) !== 'function') {
+        options.on_failure = function(){};
+    }
+    if (typeof(options.on_error) !== 'function') {
+        options.on_error = function(){};
+    }
+    if (typeof(options.on_other) !== 'function') {
+        options.on_other = function(){};
+    }
+
+    function handle_status(data) {
+        if (data === null){
+            return;
+        }
+        var task = data.task;
+        if (task === null) {
+            return;
+        }
+
+        if (task.status == 'PENDING') {
+            return;
+        }
+
+        if (task.status == 'SUCCESS') {
+            clearInterval(interval_id);
+            options.on_success(task, el);
+        }
+        else if (task.status == 'FAILURE' ) {
+            clearInterval(interval_id);
+            options.on_failure(task, el);
+        } else {
+            options.on_other(task, el);
+        }
+    }
+
+    function handle_error() {
+        ++number_of_errors;
+        // Wait after first error, just in case there is a timing issue
+        if (number_of_errors >= 2) {
+            clearInterval(interval_id);
+            options.on_error(null, el);
+        }
+    }
+
+    function check_status() {
+        $.ajax({
+            url: url,
+            data: {},
+            success: handle_status,
+            cache: false, // append a timestamp to the end of the URL
+            dataType: 'json',
+            error: handle_error
+        });
+    }
+
+    $(document).ready(function(){
+        if (url !== '') {
+            setTimeout(check_status, 0);
+            interval_id = setInterval(check_status, options.check_interval);
+        } else {
+            number_of_errors = 3;
+            handle_error();
+        }
+    });
+};

todo_django/tasks.py

@@ -0,0 +1,103 @@
+import csv
+import cStringIO
+import codecs
+import time
+
+from django.conf import settings
+
+from jobtastic import JobtasticTask
+
+from todo_django.models import Task
+
+
+class ExportTasksAsCsv(JobtasticTask):
+    """
+    Create a CSV file with all of the tasks.
+    """
+    significant_kwargs = [('task_pks', str)]
+    herd_avoidance_timeout = 120  # Give it two minutes
+    # Cache for 10 minutes if they haven't added any todos
+    cache_duration = 600
+
+    def calculate_result(self, task_pks):
+        tasks = Task.objects.filter(pk__in=task_pks)
+        num_tasks = len(task_pks)
+
+        # Let folks know we started
+        self.update_progress(0, num_tasks)
+
+        # Gather all of the data for our CSV
+        task_data = []
+        for counter, task in enumerate(tasks):
+            task_data.append([
+                task.pk, task.title, task.due_date.isoformat(),
+            ])
+
+            # Normally, we'd use an update_frequency of a couple hundred to
+            # avoid hitting the cache so often when it will only be read every
+            # couple seconds. For demo purposes though, let's wear it out!
+            self.update_progress(counter, num_tasks, update_frequency=1)
+            if getattr(settings, 'TODO_EXPORT_VERY_SLOWLY', False):
+                time.sleep(5)
+
+        # Now convert the data to CSV format
+        w = CSVWriter()
+
+        # Build the header row
+        header = ['id', 'title', 'due_date']
+        w.writerow(header)
+
+        # Now add each task's data
+        w.writerows(task_data)
+
+        # Encode the results as UTF16 for Excel's sake
+        csv_data = unicode(w.getvalue(), 'utf16')
+
+        return {'data': csv_data}
+
+
+class UnicodeWriter(object):
+    """
+    A CSV writer which will write rows to CSV file "f",
+    which is encoded in the given encoding.
+    """
+
+    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+        # Redirect output to a queue
+        self.queue = cStringIO.StringIO()
+        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
+        self.stream = f
+        self.encoder = codecs.getincrementalencoder(encoding)()
+
+    def writerow(self, row):
+        self.writer.writerow([s.encode("utf-8") for s in row])
+        # Fetch UTF-8 output from the queue ...
+        data = self.queue.getvalue()
+        data = data.decode("utf-8")
+        # ... and reencode it into the target encoding
+        data = self.encoder.encode(data)
+        # write to the target stream
+        self.stream.write(data)
+        # empty queue
+        self.queue.truncate(0)
+
+    def writerows(self, rows):
+        for row in rows:
+            self.writerow(row)
+
+
+class CSVWriter(UnicodeWriter):
+    def __init__(self):
+        self.buffer = cStringIO.StringIO()
+        super(CSVWriter, self).__init__(self.buffer, delimiter='\t')
+
+    def writerow(self, row):
+        return super(CSVWriter, self).writerow([unicode(s) for s in row])
+
+    def getvalue(self):
+        csv_data = self.buffer.getvalue()
+        # Microsoft excel does not recognize UTF-8 encoding
+        # when opening csv files. Re-encode as UTF-16.
+        csv_data = csv_data.decode('utf8').encode('utf16')
+        return csv_data
+

todo_django/templates/500.html

@@ -0,0 +1 @@
+