From f0066adccff486beb9931d9dabb05338509896a0 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 17:53:03 -0400 Subject: [PATCH 01/22] Use a proper file path for the sqlite database. --- todo_django/settings.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/todo_django/settings.py b/todo_django/settings.py index abadd0e..462ad0c 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -1,4 +1,7 @@ # Django settings for todo_django project. +import os.path + +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DEBUG = True TEMPLATE_DEBUG = DEBUG @@ -11,12 +14,12 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': 'tododjango', # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(PROJECT_ROOT, 'tododjango.db'), + 'USER': '', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', } } From eeccd78832e9f445e46792a3321378089ab94ee5 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 18:02:32 -0400 Subject: [PATCH 02/22] Add docs and celery/jobtastic requirements. --- README.md | 27 ++++++++++++++++++++++++--- deploy/django.wsgi | 4 ++++ requirements.txt | 2 ++ todo_django/settings.py | 4 ++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 387361a..2b636bd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -#Todo-Django +# Todo-Django A django web app for basic task lists that are segmented by user. -##Setup Instructions +## Setup Instructions Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2.6+ should work. - *It is recommended you use a virtual environment* - $ pip install -r requirements.txt @@ -22,7 +22,7 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2. - Single UI for most important tasks - creation, search, and deleting of tasks is all AJAX -###Includes many features inherited from Django: +### Includes many features inherited from Django: - Robust Admin - Supports multiple databases - Robust templating engine @@ -31,6 +31,27 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2. - [DirectEmployers UI Framework](https://github.com/DirectEmployers/UI-Framework) (based on Bootstrap) - [Django-Taggit](https://github.com/alex/django-taggit) +## Enhancements + +### Celery CSV report + +#### Installation + +Install the Celery and Jobtastic, +then build the database tables required by Django-Celery. + + $ pip install -r requirements.txt + $ python manage.py syncdb + +You'll also need to run a celery worker. + + $ python manage.py celery + +#### New Feature + +You now have the ability to download a CSV of all of your tasks. +That CSV is generated on Celery and gives you progress updates! + ----------------------------- # IndyPy Web Framework Shootout diff --git a/deploy/django.wsgi b/deploy/django.wsgi index df4ea16..4bb1c94 100644 --- a/deploy/django.wsgi +++ b/deploy/django.wsgi @@ -10,5 +10,9 @@ from django.conf import settings os.environ["DJANGO_SETTINGS_MODULE"] = "todo_django.settings" from django.core.handlers.wsgi import WSGIHandler + +import djcelery +djcelery.setup_loader() + application = WSGIHandler() diff --git a/requirements.txt b/requirements.txt index 2691099..f67047d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ django==1.4.3 django-taggit==0.9.3 +django-celery==3.0.17 +jobtastic==0.2.0 diff --git a/todo_django/settings.py b/todo_django/settings.py index 462ad0c..0d97e20 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -122,6 +122,7 @@ 'django.contrib.admin', 'todo_django', 'taggit', + 'djcelery', ) # A sample logging configuration. The only tangible logging @@ -152,3 +153,6 @@ }, } } + +import djcelery +djcelery.setup_loader() From 80618ad766654964c37060c7911dbcf0e28baf02 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 18:33:06 -0400 Subject: [PATCH 03/22] Configure celery to use the django broker --- README.md | 2 +- todo_django/settings.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b636bd..89d96c0 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ then build the database tables required by Django-Celery. You'll also need to run a celery worker. - $ python manage.py celery + $ python manage.py celery worker #### New Feature diff --git a/todo_django/settings.py b/todo_django/settings.py index 0d97e20..520a02b 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -123,6 +123,7 @@ 'todo_django', 'taggit', 'djcelery', + 'kombu.transport.django', ) # A sample logging configuration. The only tangible logging @@ -154,5 +155,9 @@ } } +# Use the database as Celery's broker. This is a horrible production setting, +# but makes for an easy-to-use demo +BROKER_URL = 'django://' + import djcelery djcelery.setup_loader() From 576aa2369e9187f987bfbaa272f2b34919624e60 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 18:34:39 -0400 Subject: [PATCH 04/22] Added a stubbed-out CSV task and the view to trigger it. --- todo_django/tasks.py | 14 +++++++++ todo_django/urls.py | 47 ++++++++++++++-------------- todo_django/views.py | 73 ++++++++++++++++++++++++++++---------------- 3 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 todo_django/tasks.py diff --git a/todo_django/tasks.py b/todo_django/tasks.py new file mode 100644 index 0000000..46683a3 --- /dev/null +++ b/todo_django/tasks.py @@ -0,0 +1,14 @@ +from jobtastic import JobtasticTask + +class ExportTasksAsCsv(JobtasticTask): + """ + Create a CSV file with all of the tasks. + """ + significant_kwargs = [('task_pks', str)] + herd_avoidance_timeout = 120 # Give it two minutes + # Cache for 10 minutes if they haven't added any todos + cache_duration = 600 + + def calculate_result(self, task_pks): + return {} + diff --git a/todo_django/urls.py b/todo_django/urls.py index 7810ce3..23f3db4 100644 --- a/todo_django/urls.py +++ b/todo_django/urls.py @@ -5,38 +5,37 @@ admin.autodiscover() urlpatterns = patterns('', - # Examples: - # url(r'^$', 'todo_django.views.home', name='home'), - # url(r'^todo_django/', include('todo_django.foo.urls')), - - # Uncomment the admin/doc line below to enable admin documentation: - # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: url(r'^$', 'todo_django.views.home'), - url(r'login/', - 'django.contrib.auth.views.login', + url(r'login/', + 'django.contrib.auth.views.login', {'template_name': 'index.html'} ), - url(r'logout/', - 'django.contrib.auth.views.logout', + url(r'logout/', + 'django.contrib.auth.views.logout', {'template_name': 'index.html'} - ), - url(r'^profile/tags/$', + ), + url(r'^profile/tags/$', 'todo_django.views.filter_by_tag'), - - url(r'^profile/tags/(?P.+)/$', + + url(r'^profile/tags/(?P.+)/$', 'todo_django.views.filter_by_tag'), - - url(r'^profile/add-task/$', + + url(r'^profile/add-task/$', 'todo_django.views.add_task'), - url(r'^profile/get-task/(?P\w+)/$', + url(r'^profile/get-task/(?P\w+)/$', 'todo_django.views.get_task'), - url(r'^profile/delete-task/(?P\w+)/$', + url(r'^profile/delete-task/(?P\w+)/$', 'todo_django.views.delete_task'), - - - url(r'^profile/$', 'todo_django.views.profile'), - url(r'^admin/', include(admin.site.urls)), + + + url(r'^profile/$', 'todo_django.views.profile'), + url(r'^admin/', include(admin.site.urls)), + + url(r'^profile/$', 'todo_django.views.profile'), + + url(r'^export/$', + 'todo_django.views.export_as_csv', + name='todo_django_export_as_csv' + ), ) diff --git a/todo_django/views.py b/todo_django/views.py index eb7bfc3..c8e06cb 100644 --- a/todo_django/views.py +++ b/todo_django/views.py @@ -4,23 +4,26 @@ from django.contrib.auth.decorators import login_required from django.db.models import Q from django.http import HttpResponseRedirect, HttpResponse -from django.shortcuts import render_to_response +from django.utils import simplejson as json +from django.shortcuts import render_to_response, render from django.template import RequestContext from todo_django.models import * +from todo_django.tasks import ExportTasksAsCsv from taggit.models import * + def home(request): """ Handles the root view - + Inputs: :request: Django request object - + Returns: django render_to_response object - + """ return render_to_response("index.html",{}, context_instance=RequestContext(request)) @@ -29,30 +32,30 @@ def home(request): def profile(request): """ View that handles the primary task view page. - + Inputs: :request: Django request object - + Optional Get Arguments: :search: search string used to look for matches in the title or tags :orderby: the field to order by :due: a date to match during searches - + Returns: django render_to_response object - + """ user=request.user search = request.GET.get("search") order = request.GET.get("orderby") due = request.GET.get("due") - + if not order: #set the default order to ascending date" - order = "due_date" - + order = "due_date" + if due: #convert the date to a datetime object due_dt = datetime.strptime(due,"%m/%d/%Y") - + if search and due: #search by phrase and filter by date tasks = Task.objects.get( Q(title__contains=search)|Q(tags__name__contains=search), @@ -61,14 +64,14 @@ def profile(request): elif search: #search by phrase tasks = Task.objects.filter( Q(title__contains=search)|Q(tags__name__contains=search), - user=user).order_by(order) + user=user).order_by(order) elif due: #filter by date tasks = Task.objects.filter(user=user).filter( due_date=due_dt).order_by(order) search = "Due: "+due else: #default to listing everything tasks = Task.objects.filter(user=user).order_by(order) - + ctx= { "user":user, "tasks":tasks, @@ -83,11 +86,11 @@ def delete_task(request,task_id): """ Deletes a task from the system. This view is designed to be called by AJAX. For the purposes of this demo, we assume it will always work. - + Inputs: :request: django request object :task_id: integer value of the task id - + Returns: true """ @@ -100,13 +103,13 @@ def delete_task(request,task_id): def add_task(request): """ Adds a task to the system. This is a view intended to be called by AJAX. - + Inputs: :request: django request object - + Returns: :new_task.id: The integer id of the new task - + """ task_title = request.GET.get('title') task_date = datetime.strptime(request.GET.get('date'),"%m/%d/%Y") @@ -123,11 +126,11 @@ def add_task(request): def get_task(request,task_id): """ Gets a task from the system. This view is intended to be called by AJAX. - + Inputs: :request: django request object :task_id: the integer id of the task to retrieve - + Returns: django render_to_response of a table row. """ @@ -143,17 +146,17 @@ def get_task(request,task_id): @login_required def filter_by_tag(request,tag=""): """ - Used to display a list of tags independent of tasks. It does not filter + Used to display a list of tags independent of tasks. It does not filter based on the user. - + Inputs: :request: django request object :tag: the string value of the tag to filter for. If empty, all tags are returned. - + Returns: render_to_reponse rendered template - + """ user=request.user ctx= { @@ -168,6 +171,24 @@ def filter_by_tag(request,tag=""): else: tags = Tag.objects.order_by('name') ctx["tags"] = tags - + return render_to_response(template,ctx, context_instance=RequestContext(request)) + +@login_required +def export_as_csv(request): + # Let's make a new export! + task_pks = Task.objects.all().values_list('pk', flat=True) + # Evaluate the queryset before sending it to the celery broker + task_pks = list(task_pks) + result = ExportTasksAsCsv.delay_or_fail( + task_pks=task_pks, + ) + + context = {'task_id': result.task_id} + json_content = json.dumps(context) + + return HttpResponse( + json_content, + content_type='application/json', + **httpresponse_kwargs) From afbaa96ee60475078566e78557c4e0006daf5d7a Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 18:59:27 -0400 Subject: [PATCH 05/22] Use the database cache for easy setup. --- README.md | 1 + todo_django/settings.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 89d96c0..8113508 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ then build the database tables required by Django-Celery. $ pip install -r requirements.txt $ python manage.py syncdb + $ python manage.py createcachetable cache_table You'll also need to run a celery worker. diff --git a/todo_django/settings.py b/todo_django/settings.py index 520a02b..d7c2d5a 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -155,6 +155,13 @@ } } +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'cache_table', + }, +} + # Use the database as Celery's broker. This is a horrible production setting, # but makes for an easy-to-use demo BROKER_URL = 'django://' From 829840cb3534f98e0d398ba3bf7b6604e307a4a8 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 19:01:06 -0400 Subject: [PATCH 06/22] Initial implementation of the CSV-generation task. --- todo_django/tasks.py | 83 +++++++++++++++++++++++++++++++++++++++++++- todo_django/views.py | 4 ++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/todo_django/tasks.py b/todo_django/tasks.py index 46683a3..5b365c7 100644 --- a/todo_django/tasks.py +++ b/todo_django/tasks.py @@ -1,5 +1,12 @@ +import csv +import cStringIO +import codecs + from jobtastic import JobtasticTask +from todo_django.models import Task + + class ExportTasksAsCsv(JobtasticTask): """ Create a CSV file with all of the tasks. @@ -10,5 +17,79 @@ class ExportTasksAsCsv(JobtasticTask): cache_duration = 600 def calculate_result(self, task_pks): - return {} + tasks = Task.objects.filter(pk__in=task_pks) + num_tasks = len(task_pks) + + # Let folks know we started + self.update_progress(0, num_tasks) + + # Gather all of the data for our CSV + task_data = [] + for counter, task in enumerate(tasks): + task_data.append([ + task.pk, task.title, task.due_date.isoformat(), + ]) + + self.update_progress(counter, num_tasks, update_frequency=10) + + # Now convert the data to CSV format + w = CSVWriter() + + # Build the header row + header = ['id', 'title', 'due_date'] + w.writerow(header) + + # Now add each task's data + w.writerows(task_data) + + # Encode the results as UTF16 for Excel's sake + csv_data = unicode(w.getvalue(), 'utf16') + + return {'data': csv_data} + + +class UnicodeWriter(object): + """ + A CSV writer which will write rows to CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + # Redirect output to a queue + self.queue = cStringIO.StringIO() + self.writer = csv.writer(self.queue, dialect=dialect, **kwds) + self.stream = f + self.encoder = codecs.getincrementalencoder(encoding)() + + def writerow(self, row): + self.writer.writerow([s.encode("utf-8") for s in row]) + # Fetch UTF-8 output from the queue ... + data = self.queue.getvalue() + data = data.decode("utf-8") + # ... and reencode it into the target encoding + data = self.encoder.encode(data) + # write to the target stream + self.stream.write(data) + # empty queue + self.queue.truncate(0) + + def writerows(self, rows): + for row in rows: + self.writerow(row) + + +class CSVWriter(UnicodeWriter): + def __init__(self): + self.buffer = cStringIO.StringIO() + super(CSVWriter, self).__init__(self.buffer, delimiter='\t') + + def writerow(self, row): + return super(CSVWriter, self).writerow([unicode(s) for s in row]) + + def getvalue(self): + csv_data = self.buffer.getvalue() + # Microsoft excel does not recognize UTF-8 encoding + # when opening csv files. Re-encode as UTF-16. + csv_data = csv_data.decode('utf8').encode('utf16') + return csv_data diff --git a/todo_django/views.py b/todo_django/views.py index c8e06cb..c3fab55 100644 --- a/todo_django/views.py +++ b/todo_django/views.py @@ -178,7 +178,9 @@ def filter_by_tag(request,tag=""): @login_required def export_as_csv(request): # Let's make a new export! - task_pks = Task.objects.all().values_list('pk', flat=True) + task_pks = Task.objects.filter( + user=request.user, + ).values_list('pk', flat=True) # Evaluate the queryset before sending it to the celery broker task_pks = list(task_pks) result = ExportTasksAsCsv.delay_or_fail( From c10156b42943341e590f1a84a0411a67a6185745 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 23 Apr 2013 21:22:51 -0400 Subject: [PATCH 07/22] Allow downloading the CSV result as a CSV file. --- todo_django/urls.py | 4 ++++ todo_django/views.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/todo_django/urls.py b/todo_django/urls.py index 23f3db4..e443568 100644 --- a/todo_django/urls.py +++ b/todo_django/urls.py @@ -37,5 +37,9 @@ 'todo_django.views.export_as_csv', name='todo_django_export_as_csv' ), + url(r'^export/(?P[\w\d\-]+)/$', + 'todo_django.views.export_as_csv', + name='todo_django_export_as_csv' + ), ) diff --git a/todo_django/views.py b/todo_django/views.py index c3fab55..1a7b3eb 100644 --- a/todo_django/views.py +++ b/todo_django/views.py @@ -4,14 +4,17 @@ from django.contrib.auth.decorators import login_required from django.db.models import Q from django.http import HttpResponseRedirect, HttpResponse -from django.utils import simplejson as json from django.shortcuts import render_to_response, render from django.template import RequestContext +from django.utils import simplejson as json -from todo_django.models import * +import celery +from celery.result import AsyncResult +from taggit.models import * + +from todo_django.models import Task from todo_django.tasks import ExportTasksAsCsv -from taggit.models import * def home(request): @@ -176,7 +179,20 @@ def filter_by_tag(request,tag=""): context_instance=RequestContext(request)) @login_required -def export_as_csv(request): +def export_as_csv(request, task_id=None): + if task_id: + # The export task is complete. Let's display the results. + result = AsyncResult(task_id) + if result.status != celery.states.SUCCESS: + return HttpResponseBadRequest("Export didn't succeed") + if result.result is None: + return HttpResponseBadRequest("Export has no result") + + response = HttpResponse(result.result['data'], mimetype='text/csv') + response['Content-Disposition'] = 'attachment; filename=my_tasks.csv' + + return response + # Let's make a new export! task_pks = Task.objects.filter( user=request.user, @@ -193,4 +209,4 @@ def export_as_csv(request): return HttpResponse( json_content, content_type='application/json', - **httpresponse_kwargs) + ) From dc456d20f84882c8da653c391e0a68b77ca21147 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 16:42:19 -0400 Subject: [PATCH 08/22] Template lint and whitespace cleanup --- todo_django/templates/profile.html | 43 ++++++++++---------- todo_django/templates/site_base.html | 59 ++++++++++++++-------------- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/todo_django/templates/profile.html b/todo_django/templates/profile.html index 6b706de..29ae352 100644 --- a/todo_django/templates/profile.html +++ b/todo_django/templates/profile.html @@ -9,9 +9,9 @@ url:"/profile/delete-task/"+task+"/", success: function(){ $("#task-"+task).fadeOut(500); - setTimeout($("#task-"+task).remove(),1000); + setTimeout($("#task-"+task).remove(),1000); } - }); + }); } function add_task(){ title=$("#newtask-title").val(); @@ -31,18 +31,19 @@ url:"/profile/get-task/"+task_id+"/", success: function(result){ $("#task_table").append(result); - $("#task-"+task_id).fadeIn(500); + $("#task-"+task_id).fadeIn(500); } }); } {% endblock extra_header %} {% block body %} -
-
+
+
+
-

{{user}}'s Tasks

+

{{user}}'s Tasks

@@ -62,7 +63,7 @@

{{user}}'s Tasks

{% endif %} {% endif %} - +
@@ -76,7 +77,7 @@

{{user}}'s Tasks

{% else %} {{task.due_date}} {% endif %} - {% endfor %} -
TaskDue DateTags (View All)
+ {% for tag in task.tags.all %} {{tag}} {%endfor%} @@ -84,9 +85,9 @@

{{user}}'s Tasks

+
-
+

Search Tasks

@@ -96,7 +97,7 @@

Search Tasks

- +
@@ -104,7 +105,7 @@

Search Tasks

placeholder = "due date" class="datefield" />
- +
@@ -116,7 +117,7 @@

Search Tasks

- +
@@ -127,9 +128,9 @@

Search Tasks

- +
-
+

Add task

@@ -138,32 +139,32 @@

Add task

- +
- +
- +
-
+
-
-
+ + {% endblock body %} diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index 54a53aa..d9e095f 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -4,12 +4,12 @@ IndyPy: Web Shootout - + -{% endblock extra_header %} {% block body %}
-

{{user}}'s Tasks

+

{{ user }}'s Tasks

- {% if search %} - {% if tasks %} -
- Showing results for "{{search}}" -
- Clear search -
-
-
- {% else %} -
- There were no results for "{{search}}" -
- {% endif %} - {% endif %} + {% if search %} + {% if tasks %} +
+ Showing results for "{{ search }}" +
+ Clear search +
+
+
+ {% else %} +
+ There were no results for "{{ search }}" +
+ {% endif %} + {% endif %} - - - + + + + + {% for task in tasks %} - - + + + + + + {% endfor %}
TaskDue DateTags (View All)
TaskDue DateTags (View All)
- {{task.title}} - +
+ {{ task.title }} + {% if task.due_date == now %} - {{task.due_date}} + {{ task.due_date }} {% else %} - {{task.due_date}} + {{ task.due_date }} {% endif %} - - {% for tag in task.tags.all %} - {{tag}} - {%endfor%} - - -
+ {% for tag in task.tags.all %} + {{ tag }} + {% endfor %} + + +
@@ -132,7 +101,7 @@

Search Tasks

Add task

-
+
@@ -168,3 +137,60 @@

Add task

{% endblock body %} + +{% block extra_js %} + +{% endblock extra_js %} From acda90484c51aa4a557feaad56126b6d6b114ffe Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 17:16:21 -0400 Subject: [PATCH 10/22] Stop using synchronous loading at the top of the page. Might as well just load jquery and jquery ui directly, and do it at the bottom of the page for rendering speed. --- todo_django/templates/site_base.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index d9e095f..660db9d 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -4,7 +4,6 @@ IndyPy: Web Shootout - + + {% endblock main_js %} {% block extra_js %}{% endblock extra_js %} From 7a475738b4b74b18615d88666440acfb2acaa1d0 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 17:43:15 -0400 Subject: [PATCH 11/22] DRY up task display. --- todo_django/templates/profile.html | 26 +++----------------------- todo_django/templates/single_task.html | 26 +++++++++++++++++--------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/todo_django/templates/profile.html b/todo_django/templates/profile.html index a6179df..eb01570 100644 --- a/todo_django/templates/profile.html +++ b/todo_django/templates/profile.html @@ -32,27 +32,7 @@

{{ user }}'s Tasks

Tags (View All) {% for task in tasks %} - - - {{ task.title }} - - - {% if task.due_date == now %} - {{ task.due_date }} - {% else %} - {{ task.due_date }} - {% endif %} - - - {% for tag in task.tags.all %} - {{ tag }} - {% endfor %} - - - - - + {% include "single_task.html" %} {% endfor %} @@ -183,7 +163,7 @@

Add task

$(document).ready(function() { $(".datefield").datepicker(); - $(".delete-task-btn").click(function() { + $(".delete-task-btn").live("click", function() { delete_task($(this).attr("data-task-id")); }); @@ -191,6 +171,6 @@

Add task

add_task(); return false; }); -} +}); {% endblock extra_js %} diff --git a/todo_django/templates/single_task.html b/todo_django/templates/single_task.html index 4723b26..82b6753 100644 --- a/todo_django/templates/single_task.html +++ b/todo_django/templates/single_task.html @@ -1,13 +1,21 @@ - + + + {{ task.title }} + + + {% if task.due_date == now %} + {{ task.due_date }} + {% else %} + {{ task.due_date }} + {% endif %} + - {{task.title}} - - {{task.due_date}} - {% for tag in task.tags.all %} - {{tag}} - {%endfor%} - - + {{ tag }} + {% endfor %} + + + From cf751684367d4fddc6f05685092fc543ee109a15 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 18:35:26 -0400 Subject: [PATCH 12/22] Hook in the interface to load the CSV via AJAX with a progress bar. --- todo_django/templates/site_base.html | 77 +++++++++++++++++++++++++++- todo_django/urls.py | 2 + 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index 660db9d..f339ceb 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -70,7 +70,8 @@

Direct Employers

{% if user.is_authenticated %} {{user}}'s Tasks - tags + Tags + Task Export {% endif %}
@@ -94,6 +95,80 @@

Direct Employers

{% block main_js %} + + + {% endblock main_js %} {% block extra_js %}{% endblock extra_js %} diff --git a/todo_django/urls.py b/todo_django/urls.py index e443568..5fb9ead 100644 --- a/todo_django/urls.py +++ b/todo_django/urls.py @@ -41,5 +41,7 @@ 'todo_django.views.export_as_csv', name='todo_django_export_as_csv' ), + + url(r'^task/', include('djcelery.urls')), ) From 2e3c7a181c76add8bf8be030c7ad3cbaf7859048 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 18:57:46 -0400 Subject: [PATCH 13/22] Add a delay to the export so we can see the progress working. --- todo_django/settings.py | 12 ++++++++---- todo_django/tasks.py | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/todo_django/settings.py b/todo_django/settings.py index d7c2d5a..38b3db1 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -1,9 +1,9 @@ # Django settings for todo_django project. -import os.path +import os PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -DEBUG = True +DEBUG = os.environ.get('DJANGO_DEBUG', False) TEMPLATE_DEBUG = DEBUG ADMINS = ( @@ -157,8 +157,8 @@ CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', - 'LOCATION': 'cache_table', + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': '/tmp/todo_django_cache', }, } @@ -168,3 +168,7 @@ import djcelery djcelery.setup_loader() + +# For progress demo purposes, do we want to take several seconds between each +# task for the export? +TODO_EXPORT_VERY_SLOWLY = True diff --git a/todo_django/tasks.py b/todo_django/tasks.py index 5b365c7..ba35cf3 100644 --- a/todo_django/tasks.py +++ b/todo_django/tasks.py @@ -1,6 +1,9 @@ import csv import cStringIO import codecs +import time + +from django.conf import settings from jobtastic import JobtasticTask @@ -26,6 +29,8 @@ def calculate_result(self, task_pks): # Gather all of the data for our CSV task_data = [] for counter, task in enumerate(tasks): + if getattr(settings, 'TODO_EXPORT_VERY_SLOWLY', False): + time.sleep(5) task_data.append([ task.pk, task.title, task.due_date.isoformat(), ]) From d41221a67911e3e01033f99beeded1bc2661aaeb Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 18:58:04 -0400 Subject: [PATCH 14/22] If there's a task error, replace the link to try again. --- todo_django/templates/500.html | 1 + todo_django/templates/site_base.html | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 todo_django/templates/500.html diff --git a/todo_django/templates/500.html b/todo_django/templates/500.html new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/todo_django/templates/500.html @@ -0,0 +1 @@ + diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index f339ceb..fd05eb7 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -119,8 +119,14 @@

Direct Employers

// Handler in case something goes wrong with the celery // task function error() { - var msg = '

There was an error generating your report.

'; + var msg = '

There was an error generating your report.

'; $('.progress_container').replaceWith(msg); + + // Give the user 5 seconds to see the message and then + // give them the link back + setTimeout(function() { + $('.task_error').replaceWith(originalHtml); + }, 5000); } // First, just get the task_id for the Celery task that From 82aa31f8c9356914cfc38491846860848ff65051 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 19:36:53 -0400 Subject: [PATCH 15/22] Move the progress update for quicker feedback. --- todo_django/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/todo_django/tasks.py b/todo_django/tasks.py index ba35cf3..9d9a930 100644 --- a/todo_django/tasks.py +++ b/todo_django/tasks.py @@ -29,13 +29,16 @@ def calculate_result(self, task_pks): # Gather all of the data for our CSV task_data = [] for counter, task in enumerate(tasks): - if getattr(settings, 'TODO_EXPORT_VERY_SLOWLY', False): - time.sleep(5) task_data.append([ task.pk, task.title, task.due_date.isoformat(), ]) - self.update_progress(counter, num_tasks, update_frequency=10) + # Normally, we'd use an update_frequency of a couple hundred to + # avoid hitting the cache so often when it will only be read every + # couple seconds. For demo purposes though, let's wear it out! + self.update_progress(counter, num_tasks, update_frequency=1) + if getattr(settings, 'TODO_EXPORT_VERY_SLOWLY', False): + time.sleep(5) # Now convert the data to CSV format w = CSVWriter() From 85b30d5f1f203e55118b36649a0b9cfa64072caf Mon Sep 17 00:00:00 2001 From: winhamwr Date: Wed, 24 Apr 2013 19:41:06 -0400 Subject: [PATCH 16/22] Not using the database cache anymore --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8113508..89d96c0 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ then build the database tables required by Django-Celery. $ pip install -r requirements.txt $ python manage.py syncdb - $ python manage.py createcachetable cache_table You'll also need to run a celery worker. From aaae92915648f955fda8bfadf1924e61ffc3ecf4 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Thu, 25 Apr 2013 18:18:56 -0400 Subject: [PATCH 17/22] Document the csv export in the readme. --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 89d96c0..7d0f1cb 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,19 @@ Todo-Django was developed on Ubuntu 12, but any Linux environment with python 2. ### Celery CSV report +You now have the ability to download a CSV of all of your tasks! +The CSV is generated in the background using +[Celery](http://www.celeryproject.org/). +We're also using [Jobtastic](http://policystat.github.io/jobtastic/) +on the python side to add easy progress reporting and caching. +On the client side, we're using +[jquery-celery](https://github.com/PolicyStat/jquery-celery) +to provide a Bootstrap progress bar for friendly reporting of the CSV progress. + #### Installation -Install the Celery and Jobtastic, +Install [Celery](http://www.celeryproject.org/) +and [Jobtastic](http://policystat.github.io/jobtastic/), then build the database tables required by Django-Celery. $ pip install -r requirements.txt @@ -47,10 +57,6 @@ You'll also need to run a celery worker. $ python manage.py celery worker -#### New Feature - -You now have the ability to download a CSV of all of your tasks. -That CSV is generated on Celery and gives you progress updates! ----------------------------- # IndyPy Web Framework Shootout From 750933cfc204e7e0fe849ee289e7034e7c430d1f Mon Sep 17 00:00:00 2001 From: winhamwr Date: Thu, 25 Apr 2013 18:19:10 -0400 Subject: [PATCH 18/22] Update to jobtastic 0.2.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f67047d..fbfc156 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ django==1.4.3 django-taggit==0.9.3 django-celery==3.0.17 -jobtastic==0.2.0 +jobtastic==0.2.1 From c06025625e1afee551bc5e5002b1e30e2fa3f1d1 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Thu, 25 Apr 2013 18:19:58 -0400 Subject: [PATCH 19/22] Configured static files. --- todo_django/settings.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/todo_django/settings.py b/todo_django/settings.py index 38b3db1..b18c681 100644 --- a/todo_django/settings.py +++ b/todo_django/settings.py @@ -59,7 +59,7 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = '' +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') # URL prefix for static files. # Example: "http://media.lawrence.com/static/" @@ -67,9 +67,7 @@ # Additional locations of static files STATICFILES_DIRS = ( - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. + os.path.join(PROJECT_ROOT, 'todo_django', 'static'), ) # List of finder classes that know how to find static files in @@ -77,7 +75,6 @@ STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) # Make this unique, and don't share it with anybody. @@ -87,7 +84,6 @@ TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( @@ -106,9 +102,6 @@ WSGI_APPLICATION = 'todo_django.wsgi.application' TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. ) INSTALLED_APPS = ( From 97981091d76451f5b694daee439bc2c5d22ba83f Mon Sep 17 00:00:00 2001 From: winhamwr Date: Thu, 25 Apr 2013 18:21:59 -0400 Subject: [PATCH 20/22] Switched to using a local, statically-served version of jquery-celery. --- todo_django/static/js/jquery-celery/celery.js | 74 +++++++++++++++++++ todo_django/templates/site_base.html | 4 +- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 todo_django/static/js/jquery-celery/celery.js diff --git a/todo_django/static/js/jquery-celery/celery.js b/todo_django/static/js/jquery-celery/celery.js new file mode 100644 index 0000000..c32b6f5 --- /dev/null +++ b/todo_django/static/js/jquery-celery/celery.js @@ -0,0 +1,74 @@ +jQuery.fn.djcelery = function(options) { + var interval_id; + var el = this; + var url = '/task/' + options.task_id + '/status/'; + var number_of_errors = 0; + if (options.task_id.length !== 36) { + url = ''; + } + + if (typeof(options.on_failure) !== 'function') { + options.on_failure = function(){}; + } + if (typeof(options.on_error) !== 'function') { + options.on_error = function(){}; + } + if (typeof(options.on_other) !== 'function') { + options.on_other = function(){}; + } + + function handle_status(data) { + if (data === null){ + return; + } + var task = data.task; + if (task === null) { + return; + } + + if (task.status == 'PENDING') { + return; + } + + if (task.status == 'SUCCESS') { + clearInterval(interval_id); + options.on_success(task, el); + } + else if (task.status == 'FAILURE' ) { + clearInterval(interval_id); + options.on_failure(task, el); + } else { + options.on_other(task, el); + } + } + + function handle_error() { + ++number_of_errors; + // Wait after first error, just in case there is a timing issue + if (number_of_errors >= 2) { + clearInterval(interval_id); + options.on_error(null, el); + } + } + + function check_status() { + $.ajax({ + url: url, + data: {}, + success: handle_status, + cache: false, // append a timestamp to the end of the URL + dataType: 'json', + error: handle_error + }); + } + + $(document).ready(function(){ + if (url !== '') { + setTimeout(check_status, 0); + interval_id = setInterval(check_status, options.check_interval); + } else { + number_of_errors = 3; + handle_error(); + } + }); +}; diff --git a/todo_django/templates/site_base.html b/todo_django/templates/site_base.html index fd05eb7..7a10f1e 100644 --- a/todo_django/templates/site_base.html +++ b/todo_django/templates/site_base.html @@ -1,3 +1,4 @@ +{% load staticfiles %} @@ -95,8 +96,7 @@

Direct Employers

{% block main_js %} - - +