diff --git a/.gitignore b/.gitignore index da361ea..c650c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -tododjango -todo_django/directtodo +tododjango.db +/static/ *.pyc -*.swp diff --git a/README.md b/README.md index 387361a..7d0f1cb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -#Todo-Django +# Todo-Django A django web app for basic task lists that are segmented by user. -##Setup Instructions +## Setup Instructions Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2.6+ should work. - *It is recommended you use a virtual environment* - $ pip install -r requirements.txt @@ -22,7 +22,7 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2. - Single UI for most important tasks - creation, search, and deleting of tasks is all AJAX -###Includes many features inherited from Django: +### Includes many features inherited from Django: - Robust Admin - Supports multiple databases - Robust templating engine @@ -31,6 +31,33 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2. - [DirectEmployers UI Framework](https://github.com/DirectEmployers/UI-Framework) (based on Bootstrap) - [Django-Taggit](https://github.com/alex/django-taggit) +## Enhancements + +### Celery CSV report + +You now have the ability to download a CSV of all of your tasks! +The CSV is generated in the background using +[Celery](http://www.celeryproject.org/). +We're also using [Jobtastic](http://policystat.github.io/jobtastic/) +on the python side to add easy progress reporting and caching. +On the client side, we're using +[jquery-celery](https://github.com/PolicyStat/jquery-celery) +to provide a Bootstrap progress bar for friendly reporting of the CSV progress. + +#### Installation + +Install [Celery](http://www.celeryproject.org/) +and [Jobtastic](http://policystat.github.io/jobtastic/), +then build the database tables required by Django-Celery. + + $ pip install -r requirements.txt + $ python manage.py syncdb + +You'll also need to run a celery worker. + + $ python manage.py celery worker + + ----------------------------- # IndyPy Web Framework Shootout diff --git a/deploy/django.wsgi b/deploy/django.wsgi index df4ea16..4bb1c94 100644 --- a/deploy/django.wsgi +++ b/deploy/django.wsgi @@ -10,5 +10,9 @@ from django.conf import settings os.environ["DJANGO_SETTINGS_MODULE"] = "todo_django.settings" from django.core.handlers.wsgi import WSGIHandler + +import djcelery +djcelery.setup_loader() + application = WSGIHandler() diff --git a/requirements.txt b/requirements.txt index 2691099..fbfc156 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ django==1.4.3 django-taggit==0.9.3 +django-celery==3.0.17 +jobtastic==0.2.1 diff --git a/todo_django/settings.py b/todo_django/settings.py index abadd0e..b18c681 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -1,6 +1,9 @@ # Django settings for todo_django project. +import os -DEBUG = True +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +DEBUG = os.environ.get('DJANGO_DEBUG', False) TEMPLATE_DEBUG = DEBUG ADMINS = ( @@ -11,12 +14,12 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': 'tododjango', # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(PROJECT_ROOT, 'tododjango.db'), + 'USER': '', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', } } @@ -56,7 +59,7 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = '' +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') # URL prefix for static files. # Example: "http://media.lawrence.com/static/" @@ -64,9 +67,7 @@ # Additional locations of static files STATICFILES_DIRS = ( - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. + os.path.join(PROJECT_ROOT, 'todo_django', 'static'), ) # List of finder classes that know how to find static files in @@ -74,7 +75,6 @@ STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) # Make this unique, and don't share it with anybody. @@ -84,7 +84,6 @@ TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( @@ -103,9 +102,6 @@ WSGI_APPLICATION = 'todo_django.wsgi.application' TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. ) INSTALLED_APPS = ( @@ -119,6 +115,8 @@ 'django.contrib.admin', 'todo_django', 'taggit', + 'djcelery', + 'kombu.transport.django', ) # A sample logging configuration. The only tangible logging @@ -149,3 +147,21 @@ }, } } + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': '/tmp/todo_django_cache', + }, +} + +# Use the database as Celery's broker. This is a horrible production setting, +# but makes for an easy-to-use demo +BROKER_URL = 'django://' + +import djcelery +djcelery.setup_loader() + +# For progress demo purposes, do we want to take several seconds between each +# task for the export? +TODO_EXPORT_VERY_SLOWLY = True diff --git a/todo_django/static/js/jquery-celery/celery.js b/todo_django/static/js/jquery-celery/celery.js new file mode 100644 index 0000000..c32b6f5 --- /dev/null +++ b/todo_django/static/js/jquery-celery/celery.js @@ -0,0 +1,74 @@ +jQuery.fn.djcelery = function(options) { + var interval_id; + var el = this; + var url = '/task/' + options.task_id + '/status/'; + var number_of_errors = 0; + if (options.task_id.length !== 36) { + url = ''; + } + + if (typeof(options.on_failure) !== 'function') { + options.on_failure = function(){}; + } + if (typeof(options.on_error) !== 'function') { + options.on_error = function(){}; + } + if (typeof(options.on_other) !== 'function') { + options.on_other = function(){}; + } + + function handle_status(data) { + if (data === null){ + return; + } + var task = data.task; + if (task === null) { + return; + } + + if (task.status == 'PENDING') { + return; + } + + if (task.status == 'SUCCESS') { + clearInterval(interval_id); + options.on_success(task, el); + } + else if (task.status == 'FAILURE' ) { + clearInterval(interval_id); + options.on_failure(task, el); + } else { + options.on_other(task, el); + } + } + + function handle_error() { + ++number_of_errors; + // Wait after first error, just in case there is a timing issue + if (number_of_errors >= 2) { + clearInterval(interval_id); + options.on_error(null, el); + } + } + + function check_status() { + $.ajax({ + url: url, + data: {}, + success: handle_status, + cache: false, // append a timestamp to the end of the URL + dataType: 'json', + error: handle_error + }); + } + + $(document).ready(function(){ + if (url !== '') { + setTimeout(check_status, 0); + interval_id = setInterval(check_status, options.check_interval); + } else { + number_of_errors = 3; + handle_error(); + } + }); +}; diff --git a/todo_django/tasks.py b/todo_django/tasks.py new file mode 100644 index 0000000..9d9a930 --- /dev/null +++ b/todo_django/tasks.py @@ -0,0 +1,103 @@ +import csv +import cStringIO +import codecs +import time + +from django.conf import settings + +from jobtastic import JobtasticTask + +from todo_django.models import Task + + +class ExportTasksAsCsv(JobtasticTask): + """ + Create a CSV file with all of the tasks. + """ + significant_kwargs = [('task_pks', str)] + herd_avoidance_timeout = 120 # Give it two minutes + # Cache for 10 minutes if they haven't added any todos + cache_duration = 600 + + def calculate_result(self, task_pks): + tasks = Task.objects.filter(pk__in=task_pks) + num_tasks = len(task_pks) + + # Let folks know we started + self.update_progress(0, num_tasks) + + # Gather all of the data for our CSV + task_data = [] + for counter, task in enumerate(tasks): + task_data.append([ + task.pk, task.title, task.due_date.isoformat(), + ]) + + # Normally, we'd use an update_frequency of a couple hundred to + # avoid hitting the cache so often when it will only be read every + # couple seconds. For demo purposes though, let's wear it out! + self.update_progress(counter, num_tasks, update_frequency=1) + if getattr(settings, 'TODO_EXPORT_VERY_SLOWLY', False): + time.sleep(5) + + # Now convert the data to CSV format + w = CSVWriter() + + # Build the header row + header = ['id', 'title', 'due_date'] + w.writerow(header) + + # Now add each task's data + w.writerows(task_data) + + # Encode the results as UTF16 for Excel's sake + csv_data = unicode(w.getvalue(), 'utf16') + + return {'data': csv_data} + + +class UnicodeWriter(object): + """ + A CSV writer which will write rows to CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + # Redirect output to a queue + self.queue = cStringIO.StringIO() + self.writer = csv.writer(self.queue, dialect=dialect, **kwds) + self.stream = f + self.encoder = codecs.getincrementalencoder(encoding)() + + def writerow(self, row): + self.writer.writerow([s.encode("utf-8") for s in row]) + # Fetch UTF-8 output from the queue ... + data = self.queue.getvalue() + data = data.decode("utf-8") + # ... and reencode it into the target encoding + data = self.encoder.encode(data) + # write to the target stream + self.stream.write(data) + # empty queue + self.queue.truncate(0) + + def writerows(self, rows): + for row in rows: + self.writerow(row) + + +class CSVWriter(UnicodeWriter): + def __init__(self): + self.buffer = cStringIO.StringIO() + super(CSVWriter, self).__init__(self.buffer, delimiter='\t') + + def writerow(self, row): + return super(CSVWriter, self).writerow([unicode(s) for s in row]) + + def getvalue(self): + csv_data = self.buffer.getvalue() + # Microsoft excel does not recognize UTF-8 encoding + # when opening csv files. Re-encode as UTF-16. + csv_data = csv_data.decode('utf8').encode('utf16') + return csv_data + diff --git a/todo_django/templates/500.html b/todo_django/templates/500.html new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/todo_django/templates/500.html @@ -0,0 +1 @@ + diff --git a/todo_django/templates/profile.html b/todo_django/templates/profile.html index 6b706de..eb01570 100644 --- a/todo_django/templates/profile.html +++ b/todo_django/templates/profile.html @@ -1,92 +1,42 @@ {% extends "site_base.html" %} -{% block extra_header %} - -{% endblock extra_header %} {% block body %} -
-
+
+
+
-

{{user}}'s Tasks

+

{{ user }}'s Tasks

- {% if search %} - {% if tasks %} -
- Showing results for "{{search}}" - -
-
- {% else %} -
- There were no results for "{{search}}" -
- {% endif %} - {% endif %} - - - - + {% if search %} + {% if tasks %} +
+ Showing results for "{{ search }}" + +
+
+ {% else %} +
+ There were no results for "{{ search }}" +
+ {% endif %} + {% endif %} +
TaskDue DateTags (View All)
+ + + + + {% for task in tasks %} - - + {% include "single_task.html" %} {% endfor %} -
TaskDue DateTags (View All)
- {{task.title}} - - {% if task.due_date == now %} - {{task.due_date}} - {% else %} - {{task.due_date}} - {% endif %} - - {% for tag in task.tags.all %} - {{tag}} - {%endfor%} - - -
+
-
+

Search Tasks

@@ -96,7 +46,7 @@

Search Tasks

- +
@@ -104,7 +54,7 @@

Search Tasks

placeholder = "due date" class="datefield" />
- +
@@ -116,7 +66,7 @@

Search Tasks

- +
@@ -127,43 +77,100 @@

Search Tasks

- +
-
+

Add task

-
+
- +
- +
- +
-
+
-
-
+ + {% endblock body %} + +{% block extra_js %} + +{% endblock extra_js %} diff --git a/todo_django/templates/single_task.html b/todo_django/templates/single_task.html index 4723b26..82b6753 100644 --- a/todo_django/templates/single_task.html +++ b/todo_django/templates/single_task.html @@ -1,13 +1,21 @@ - + + + {{ task.title }} + + + {% if task.due_date == now %} + {{ task.due_date }} + {% else %} + {{ task.due_date }} + {% endif %} + - {{task.title}} - - {{task.due_date}} - {% for tag in task.tags.all %} - {{tag}} - {%endfor%} - - + {{ tag }} + {% endfor %} + + + diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index 54a53aa..b98573a 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -1,15 +1,15 @@ +{% load staticfiles %} IndyPy: Web Shootout - - {% block extra_header %}{%endblock%} + {% block extra_header %}{% endblock extra_header %} {% block topbar %}
-
- -
- -
+
- {%endblock%} - {% block body %}{%endblock%} + {% endblock topbar %} + {% block body %}{% endblock body %} + {% block main_js %} + + + + + {% endblock main_js %} + {% block extra_js %}{% endblock extra_js %} diff --git a/todo_django/urls.py b/todo_django/urls.py index 7810ce3..5fb9ead 100644 --- a/todo_django/urls.py +++ b/todo_django/urls.py @@ -5,38 +5,43 @@ admin.autodiscover() urlpatterns = patterns('', - # Examples: - # url(r'^$', 'todo_django.views.home', name='home'), - # url(r'^todo_django/', include('todo_django.foo.urls')), - - # Uncomment the admin/doc line below to enable admin documentation: - # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: url(r'^$', 'todo_django.views.home'), - url(r'login/', - 'django.contrib.auth.views.login', + url(r'login/', + 'django.contrib.auth.views.login', {'template_name': 'index.html'} ), - url(r'logout/', - 'django.contrib.auth.views.logout', + url(r'logout/', + 'django.contrib.auth.views.logout', {'template_name': 'index.html'} - ), - url(r'^profile/tags/$', + ), + url(r'^profile/tags/$', 'todo_django.views.filter_by_tag'), - - url(r'^profile/tags/(?P.+)/$', + + url(r'^profile/tags/(?P.+)/$', 'todo_django.views.filter_by_tag'), - - url(r'^profile/add-task/$', + + url(r'^profile/add-task/$', 'todo_django.views.add_task'), - url(r'^profile/get-task/(?P\w+)/$', + url(r'^profile/get-task/(?P\w+)/$', 'todo_django.views.get_task'), - url(r'^profile/delete-task/(?P\w+)/$', + url(r'^profile/delete-task/(?P\w+)/$', 'todo_django.views.delete_task'), - - - url(r'^profile/$', 'todo_django.views.profile'), - url(r'^admin/', include(admin.site.urls)), + + + url(r'^profile/$', 'todo_django.views.profile'), + url(r'^admin/', include(admin.site.urls)), + + url(r'^profile/$', 'todo_django.views.profile'), + + url(r'^export/$', + 'todo_django.views.export_as_csv', + name='todo_django_export_as_csv' + ), + url(r'^export/(?P[\w\d\-]+)/$', + 'todo_django.views.export_as_csv', + name='todo_django_export_as_csv' + ), + + url(r'^task/', include('djcelery.urls')), ) diff --git a/todo_django/views.py b/todo_django/views.py index eb7bfc3..1a7b3eb 100644 --- a/todo_django/views.py +++ b/todo_django/views.py @@ -4,23 +4,29 @@ from django.contrib.auth.decorators import login_required from django.db.models import Q from django.http import HttpResponseRedirect, HttpResponse -from django.shortcuts import render_to_response +from django.shortcuts import render_to_response, render from django.template import RequestContext +from django.utils import simplejson as json -from todo_django.models import * - +import celery +from celery.result import AsyncResult from taggit.models import * +from todo_django.models import Task +from todo_django.tasks import ExportTasksAsCsv + + + def home(request): """ Handles the root view - + Inputs: :request: Django request object - + Returns: django render_to_response object - + """ return render_to_response("index.html",{}, context_instance=RequestContext(request)) @@ -29,30 +35,30 @@ def home(request): def profile(request): """ View that handles the primary task view page. - + Inputs: :request: Django request object - + Optional Get Arguments: :search: search string used to look for matches in the title or tags :orderby: the field to order by :due: a date to match during searches - + Returns: django render_to_response object - + """ user=request.user search = request.GET.get("search") order = request.GET.get("orderby") due = request.GET.get("due") - + if not order: #set the default order to ascending date" - order = "due_date" - + order = "due_date" + if due: #convert the date to a datetime object due_dt = datetime.strptime(due,"%m/%d/%Y") - + if search and due: #search by phrase and filter by date tasks = Task.objects.get( Q(title__contains=search)|Q(tags__name__contains=search), @@ -61,14 +67,14 @@ def profile(request): elif search: #search by phrase tasks = Task.objects.filter( Q(title__contains=search)|Q(tags__name__contains=search), - user=user).order_by(order) + user=user).order_by(order) elif due: #filter by date tasks = Task.objects.filter(user=user).filter( due_date=due_dt).order_by(order) search = "Due: "+due else: #default to listing everything tasks = Task.objects.filter(user=user).order_by(order) - + ctx= { "user":user, "tasks":tasks, @@ -83,11 +89,11 @@ def delete_task(request,task_id): """ Deletes a task from the system. This view is designed to be called by AJAX. For the purposes of this demo, we assume it will always work. - + Inputs: :request: django request object :task_id: integer value of the task id - + Returns: true """ @@ -100,13 +106,13 @@ def delete_task(request,task_id): def add_task(request): """ Adds a task to the system. This is a view intended to be called by AJAX. - + Inputs: :request: django request object - + Returns: :new_task.id: The integer id of the new task - + """ task_title = request.GET.get('title') task_date = datetime.strptime(request.GET.get('date'),"%m/%d/%Y") @@ -123,11 +129,11 @@ def add_task(request): def get_task(request,task_id): """ Gets a task from the system. This view is intended to be called by AJAX. - + Inputs: :request: django request object :task_id: the integer id of the task to retrieve - + Returns: django render_to_response of a table row. """ @@ -143,17 +149,17 @@ def get_task(request,task_id): @login_required def filter_by_tag(request,tag=""): """ - Used to display a list of tags independent of tasks. It does not filter + Used to display a list of tags independent of tasks. It does not filter based on the user. - + Inputs: :request: django request object :tag: the string value of the tag to filter for. If empty, all tags are returned. - + Returns: render_to_reponse rendered template - + """ user=request.user ctx= { @@ -168,6 +174,39 @@ def filter_by_tag(request,tag=""): else: tags = Tag.objects.order_by('name') ctx["tags"] = tags - + return render_to_response(template,ctx, context_instance=RequestContext(request)) + +@login_required +def export_as_csv(request, task_id=None): + if task_id: + # The export task is complete. Let's display the results. + result = AsyncResult(task_id) + if result.status != celery.states.SUCCESS: + return HttpResponseBadRequest("Export didn't succeed") + if result.result is None: + return HttpResponseBadRequest("Export has no result") + + response = HttpResponse(result.result['data'], mimetype='text/csv') + response['Content-Disposition'] = 'attachment; filename=my_tasks.csv' + + return response + + # Let's make a new export! + task_pks = Task.objects.filter( + user=request.user, + ).values_list('pk', flat=True) + # Evaluate the queryset before sending it to the celery broker + task_pks = list(task_pks) + result = ExportTasksAsCsv.delay_or_fail( + task_pks=task_pks, + ) + + context = {'task_id': result.task_id} + json_content = json.dumps(context) + + return HttpResponse( + json_content, + content_type='application/json', + )