Run app on heroku

.gitignore

@@ -1,2 +1,3 @@
 __pycache__/
 db.sqlite3
+/staticfiles/

.travis.yml

@@ -1,3 +1,5 @@
 language: python
+python:
+  - 3.6
 
 script: flake8

Procfile

@@ -0,0 +1 @@
+web: gunicorn ballot_api.wsgi --log-file -

README.md

@@ -9,6 +9,9 @@ California.
 
 ## Prequisites
 
+- Python 3
+- [PostgreSQL](https://www.postgresql.org/)
+
 
 ## Setup
 

ballot/admin.py

@@ -14,11 +14,7 @@ class PersonAdmin(admin.ModelAdmin):
     exclude = ('filer',)
 
 
-class BallotApiAdminSite(admin.AdminSite):
-    site_header = 'Open Disclosure Ballot API Admin'
-
-
-api_admin = BallotApiAdminSite(name='ballotapi-admin')
-api_admin.register(Election)
-api_admin.register(Person, PersonAdmin)
-api_admin.register(State)
+admin.site.site_header = 'Open Disclosure Ballot API Admin'
+admin.site.register(Election)
+admin.site.register(Person, PersonAdmin)
+admin.site.register(State)

ballot_api/settings.py → ballot_api/settings/common.py

@@ -1,13 +1,25 @@
+import logging
 import os
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+logging.basicConfig(level=logging.INFO)
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 SECRET_KEY = 'y%72v=z^op&=n4$ye^u$*n0!a=v*-qs_&)#rs*wf+m3up85)(#'
 
 DEBUG = True
 
 ALLOWED_HOSTS = []
 
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'ballot_api',
+        'HOST': '127.0.0.1',
+        'PORT': '5432',
+    }
+}
+
 INSTALLED_APPS = (
     'flat',
     'django.contrib.admin',
@@ -18,6 +30,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'ballot',
+    'gsheets',
     'rest_framework_swagger',
 )
 
@@ -31,6 +44,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
 )
 
 ROOT_URLCONF = 'ballot_api.urls'
@@ -53,13 +67,6 @@
 
 WSGI_APPLICATION = 'ballot_api.wsgi.application'
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    }
-}
-
 LANGUAGE_CODE = 'en-us'
 
 TIME_ZONE = 'UTC'
@@ -71,3 +78,5 @@
 USE_TZ = True
 
 STATIC_URL = '/static/'
+STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

ballot_api/settings/development.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+from .common import *
+
+try:
+    from local_settings import *
+except ImportError:
+    pass

ballot_api/settings/heroku.py

@@ -0,0 +1,20 @@
+# flake8: noqa
+
+import os
+
+import dj_database_url
+
+from .common import *
+
+DEBUG = False
+
+ALLOWED_HOSTS = [
+    'caciviclab-ballot-api.herokuapp.com',
+]
+
+DATABASES['default'] = dj_database_url.config()
+
+SECRET_KEY = os.environ.get('SECRET_KEY', None)
+
+if not SECRET_KEY:
+    raise Exception('SECRET_KEY environment variable must be set for heroku configuration.')

ballot_api/urls.py

@@ -1,20 +1,20 @@
 from django.conf.urls import url, include
+from django.contrib import admin
 from rest_framework import routers
 from rest_framework_swagger.views import get_swagger_view
 
-from ballot.admin import api_admin
-from ballot.views import CandidateViewSet, ElectionViewSet
+import ballot.views as ballot_views
+import gsheets.views as gsheets_views
 
-
-router = routers.DefaultRouter()
-router.register(r'candidates', CandidateViewSet)
-router.register(r'elections', ElectionViewSet)
+router = routers.SimpleRouter()
+router.register(r'elections', ballot_views.ElectionViewSet)
+router.register(r'candidates', gsheets_views.CandidateViewSet)
 
 schema_view = get_swagger_view(title='Open Disclosure Ballot API')
 
 urlpatterns = [
     url(r'^api/', include(router.urls)),
     url(r'^$', schema_view),
     url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
-    url(r'^admin/', api_admin.urls),
+    url(r'^admin/', include(admin.site.urls)),
 ]

ballot_api/wsgi.py

@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings.heroku")
 
 application = get_wsgi_application()

gsheets/admin.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+from .models import Candidate, CandidateAlias, Committee, Referendum, ReferendumMapping
+
+
+class CandidateAliasInline(admin.TabularInline):
+    model = CandidateAlias
+
+
+class CandidateAdmin(admin.ModelAdmin):
+    inlines = (CandidateAliasInline,)
+
+
+admin.site.register(Candidate, CandidateAdmin)
+admin.site.register(CandidateAlias)
+admin.site.register(Referendum)
+admin.site.register(ReferendumMapping)
+admin.site.register(Committee)

gsheets/management/commands/gsheets_import.py

@@ -0,0 +1,93 @@
+import csv
+import io
+import logging
+import re
+import tempfile
+import urllib.request
+
+from django.core.management.base import BaseCommand
+
+from gsheets import parsers
+
+
+CANDIDATES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=csv' # noqa
+REFERENDUMS_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1693935349&single=true&output=csv' # noqa
+REFERENDUM_NAME_TO_NUMBER_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=896561174&single=true&output=csv' # noqa
+COMMITTEES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1995437960&single=true&output=csv' # noqa
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Import data from Google Sheets'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            default=False,
+            help="Import the entity even if it already exists.")
+
+    def format_csv_fields(self, row):
+        """Lowercases the keys of row and removes any special characters"""
+
+        # Lowercase the keys
+        model = ((k.lower(), v) for k, v in row.items())
+
+        # Replace spaces with underscore
+        model = ((re.sub(r'\s+', '_', k), v) for k, v in model)
+
+        # Strip other characters
+        model = ((re.sub(r'[^a-z_]', '', k), v) for k, v in model)
+
+        return dict(model)
+
+    def handle(self, *args, **options):
+        self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options)
+        self.fetch_and_parse_from_url(COMMITTEES_SHEET, parsers.CommitteeParser(), **options)
+        self.fetch_and_parse_from_url(REFERENDUMS_SHEET, parsers.ReferendumParser(), **options)
+        self.fetch_and_parse_from_url(REFERENDUM_NAME_TO_NUMBER_SHEET, parsers.ReferendumMappingParser(), **options)
+
+    def fetch_and_parse_from_url(self, url, parser, force=False, **options):
+        with urllib.request.urlopen(url) as request:
+            with tempfile.TemporaryFile() as csvfile:
+                csvfile.write(request.read())
+                csvfile.seek(0)
+                reader = csv.DictReader(io.TextIOWrapper(csvfile, encoding='utf-8'))
+
+                total = 0
+                imported = 0
+                import_errors = 0
+                for row in reader:
+                    total += 1
+
+                    row = self.format_csv_fields(row)
+                    logger.debug(row)
+
+                    exists = parser.exists_in_db(row)
+                    if exists and not force:
+                        # Skip this row
+                        continue
+
+                    id = row.get(parser.key)
+
+                    try:
+                        data = parser.parse(row)
+                        logger.debug(data)
+                        model, created = parser.commit(data)
+                    except Exception as err:
+                        import_errors += 1
+                        logger.error('%s "%s" could not be parsed: parse_errors=%s row=%s',
+                                     parser.name, id, err, row)
+                        logger.exception(err)
+                        continue
+
+                    imported += 1
+                    if created:
+                        logger.info('Created %s "%s"', parser.name, id)
+                    else:
+                        logger.info('Updated %s "%s"', parser.name, id)
+
+                logger.info('Import %s data complete: total=%s imported=%s errors=%s',
+                            parser.name, total, imported, import_errors)

gsheets/migrations/0001_initial.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Candidate',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('fppc', models.IntegerField(blank=True, unique=True, null=True)),
+                ('committee_name', models.CharField(blank=True, max_length=120)),
+                ('candidate', models.CharField(max_length=30)),
+                ('office', models.CharField(blank=True, help_text='Office the candidate is running for.', max_length=30)),
+                ('incumbent', models.BooleanField(default=False)),
+                ('accepted_expenditure_ceiling', models.BooleanField(default=False)),
+                ('website', models.URLField(blank=True, null=True)),
+                ('twitter', models.URLField(blank=True, null=True)),
+                ('party_affiliation', models.CharField(blank=True, max_length=15)),
+                ('occupation', models.CharField(blank=True, max_length=80)),
+                ('bio', models.TextField(blank=True)),
+                ('photo', models.URLField(blank=True, null=True)),
+                ('votersedge', models.URLField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Committee',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('filer_id', models.CharField(max_length=15)),
+                ('filer_naml', models.CharField(blank=True, max_length=200)),
+                ('committee_type', models.CharField(choices=[('RCP', 'RCP'), ('BMC', 'BMC')], blank=True, max_length=3)),
+                ('description', models.CharField(blank=True, max_length=40)),
+                ('ballot_measure', models.CharField(blank=True, max_length=3)),
+                ('support_or_oppose', models.CharField(choices=[('S', 'Support'), ('O', 'Oppose')], blank=True, max_length=1)),
+                ('website', models.URLField(blank=True, null=True)),
+                ('twitter', models.URLField(blank=True, null=True)),
+                ('facebook', models.URLField(blank=True, null=True)),
+                ('netfilelocalid', models.CharField(blank=True, max_length=15)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Referendum',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('measure_number', models.CharField(max_length=5)),
+                ('short_title', models.CharField(blank=True, max_length=30)),
+                ('full_title', models.TextField(blank=True)),
+                ('summary', models.TextField()),
+                ('votersedge', models.URLField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ReferendumMapping',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('measure_name', models.CharField(max_length=30)),
+                ('measure_number', models.CharField(max_length=3)),
+            ],
+        ),
+    ]

gsheets/migrations/0002_auto_20170205_0455.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gsheets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CandidateAlias',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('candidate_alias', models.CharField(max_length=30)),
+                ('candidate', models.ForeignKey(to='gsheets.Candidate', related_name='aliases')),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='referendum',
+            name='short_title',
+            field=models.CharField(max_length=200, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='referendummapping',
+            name='measure_name',
+            field=models.CharField(max_length=200),
+        ),
+    ]