From 9590a0911db4774f0b410a981d0632f9fc262875 Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Sat, 20 Aug 2016 15:56:07 -0700 Subject: [PATCH 1/3] Add LOGGING configuration This enables running django management commands with DJANGO_LOG_LEVEL=DEBUG in order to see all the SQL queries that django is executing. --- disclosure/settings.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/disclosure/settings.py b/disclosure/settings.py index a177d0bd..a3e93e24 100644 --- a/disclosure/settings.py +++ b/disclosure/settings.py @@ -31,6 +31,22 @@ NETFILE_DOWNLOAD_DIR = op.join(DATA_DIR, 'netfile') +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, + }, +} + INSTALLED_APPS = ( 'django.contrib.admin', 'django.contrib.auth', From 68177343387812f56be73fee01507ae1e70ecec7 Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Sun, 21 Aug 2016 17:20:51 -0700 Subject: [PATCH 2/3] Add django command to import from the spreadsheet The spreadsheet is here: https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/edit#gid=0 This adds a django management command which downloads the (currently, only first page of the) spreadsheet in TSV format[1] and imports each row as its own OfficeElection/Candidate. python manage.py importoaklandcandidates There are some subsequent TODO's but for now I want to get this in so we can start playing with real data. Some follow-up work: * Store the FPPC id's for candidate controlled committees and whatever is equivalent for ballot measures. * Actually link up contributions with those FPPC id's * Figure out what to do with the "Non-controlled committees" spreadsheet [1]: my first attempt to parse CSV got tripped up because of commas --- .../commands/importoaklandcandidates.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 finance/management/commands/importoaklandcandidates.py diff --git a/finance/management/commands/importoaklandcandidates.py b/finance/management/commands/importoaklandcandidates.py new file mode 100644 index 00000000..58533937 --- /dev/null +++ b/finance/management/commands/importoaklandcandidates.py @@ -0,0 +1,104 @@ +import csv +import datetime +import urllib2 + +from django.core.management.base import BaseCommand + +from ballot.models import Ballot, Candidate, Office, OfficeElection +from locality.models import City, State +from ballot.models.referendum import Referendum + + +HUMAN_CANDIDATES_URL = \ + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=tsv' +BALLOT_MEASURES_URL = \ + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1693935349&single=true&output=tsv' +NON_CONTROLLED_COMMITTEE_URL = \ + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1995437960&single=true&output=tsv' + +class Command(BaseCommand): + help = 'Import the Oakland Candidates from the spreadsheet' + + def handle(self, *args, **options): + # BALLOT: + state, _ = State.objects.get_or_create(name='California', short_name='CA') + oakland, _ = City.objects.get_or_create( + name='Oakland', + short_name='COAK', + state=state + ) + + ballot, _ = Ballot.objects.get_or_create( + locality=oakland + ) + ballot.date = datetime.date(2016, 11, 6) + ballot.save() + + # HUMAN CANDIDATES + # ===================================================================== + self.process_humans(ballot, oakland) + + # BALLOT MEASURES + # ===================================================================== + self.process_ballot_measures(ballot) + + def process_ballot_measures(self, ballot): + resp = urllib2.urlopen(BALLOT_MEASURES_URL) + rows = csv.DictReader(resp, delimiter='\t') + title_column = 'Ballot Measures: ' \ + 'https://drive.google.com/open?id=0BzmHYSKNqcR_TjdjY21icWt6VUE' + for row in rows: + referendum, created = Referendum.objects.get_or_create( + title=row[title_column], + number=row['Measure alpha-numeric designation'], + contest_type='R', + ballot=ballot + ) + if created: + print 'created!' + else: + print referendum + resp.close() + + def process_humans(self, ballot, oakland): + resp = urllib2.urlopen(HUMAN_CANDIDATES_URL) + rows = csv.DictReader(resp, delimiter='\t') + for row in rows: + # TODO: Update the spreadsheet to handle split names + names = row['Candidate'].split(' ', 3) + if len(names) == 2: + first_name = names[0] + middle_name = '' + last_name = names[1] + elif len(names) > 2: + first_name = names[0] + middle_name = " ".join(names[1:-1]) + last_name = " ".join(names[-1:]) + + candidate_office, _ = Office.objects.get_or_create( + name=row['Office'], + locality=oakland + ) + office_election, _ = OfficeElection.objects.get_or_create( + office=candidate_office, + ballot=ballot + ) + + candidate = Candidate.objects.filter( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + office_election=office_election + ) + + if candidate: + print "Found existing candidate: " + str(candidate) + else: + candidate.get_or_create( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + office_election=office_election + ) + print "Creating candidate: " + str(candidate) + resp.close() From 836b27b1bb1f739ed779fc1aa6eae9af0097e8f8 Mon Sep 17 00:00:00 2001 From: Tom Dooner Date: Mon, 22 Aug 2016 22:08:25 -0700 Subject: [PATCH 3/3] Fix flake8 in importoaklandcandidates.py --- finance/management/commands/importoaklandcandidates.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/finance/management/commands/importoaklandcandidates.py b/finance/management/commands/importoaklandcandidates.py index 58533937..df25f54e 100644 --- a/finance/management/commands/importoaklandcandidates.py +++ b/finance/management/commands/importoaklandcandidates.py @@ -10,11 +10,15 @@ HUMAN_CANDIDATES_URL = \ - 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=tsv' + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI' \ + '/pub?gid=0&single=true&output=tsv' BALLOT_MEASURES_URL = \ - 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1693935349&single=true&output=tsv' + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI' \ + '/pub?gid=1693935349&single=true&output=tsv' NON_CONTROLLED_COMMITTEE_URL = \ - 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1995437960&single=true&output=tsv' + 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI' \ + '/pub?gid=1995437960&single=true&output=tsv' + class Command(BaseCommand): help = 'Import the Oakland Candidates from the spreadsheet'