From ec1478b72cbfebd5e8c9ba8cc33153f41482bc93 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Tue, 19 Jul 2016 20:12:46 -0700 Subject: [PATCH 01/14] Remove ReportingPeriod models and fixtures --- finance/admin.py | 3 +- finance/management/commands/resolve_money.py | 2 +- finance/migrations/0001_initial.py | 19 +--------- finance/migrations/0004_auto_20160227_1512.py | 19 ---------- finance/models.py | 35 +------------------ finance/tests/test_strings.py | 4 +-- 6 files changed, 6 insertions(+), 76 deletions(-) delete mode 100644 finance/migrations/0004_auto_20160227_1512.py diff --git a/finance/admin.py b/finance/admin.py index 14735b76..4ec03c7d 100644 --- a/finance/admin.py +++ b/finance/admin.py @@ -32,12 +32,11 @@ class BeneficiaryAdmin(admin.ModelAdmin): class IndependentMoneyAdmin(admin.ModelAdmin): readonly_fields = ('benefactor', 'benefactor_zip', 'beneficiary', 'amount', 'cumulative_amount', 'report_date', - 'reporting_period', 'source', 'source_xact_id') + 'source', 'source_xact_id') admin.site.register(models.Committee) admin.site.register(models.Employer) admin.site.register(models.Form) -admin.site.register(models.ReportingPeriod) validate_and_register_admin( models.Beneficiary, BeneficiaryAdmin, num_hidden_fields=5) diff --git a/finance/management/commands/resolve_money.py b/finance/management/commands/resolve_money.py index 33d99d30..201aa5ef 100644 --- a/finance/management/commands/resolve_money.py +++ b/finance/management/commands/resolve_money.py @@ -22,5 +22,5 @@ def handle(self, *args, **options): print related_money.index(im), str(im) for rm in related_money: print "\t%s %10s %s %s %s" % ( - rm.report_date, str(rm.reporting_period)[:10], rm.filing_id, + rm.report_date, rm.filing_id, rm.source_xact_id, rm.benefactor) diff --git a/finance/migrations/0001_initial.py b/finance/migrations/0001_initial.py index c171dde2..0515f94f 100644 --- a/finance/migrations/0001_initial.py +++ b/finance/migrations/0001_initial.py @@ -65,22 +65,10 @@ class Migration(migrations.Migration): ('source_xact_id', models.CharField(help_text='Transaction ID (specific to data source)', max_length=32)), ], options={ - 'ordering': ('-reporting_period__period_start', '-reporting_period__period_end', '-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), + 'ordering': ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), 'verbose_name_plural': 'independent money', }, ), - migrations.CreateModel( - name='ReportingPeriod', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('period_start', models.DateField()), - ('period_end', models.DateField()), - ('form', models.ForeignKey(to='finance.Form')), - ], - options={ - 'ordering': ('period_start', 'period_end'), - }, - ), migrations.CreateModel( name='Beneficiary', fields=[ @@ -169,11 +157,6 @@ class Migration(migrations.Migration): name='benefactor_zip', field=models.ForeignKey(to='locality.ZipCode'), ), - migrations.AddField( - model_name='independentmoney', - name='reporting_period', - field=models.ForeignKey(to='finance.ReportingPeriod'), - ), migrations.AddField( model_name='committee', name='city', diff --git a/finance/migrations/0004_auto_20160227_1512.py b/finance/migrations/0004_auto_20160227_1512.py deleted file mode 100644 index 645209de..00000000 --- a/finance/migrations/0004_auto_20160227_1512.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('finance', '0003_merge'), - ] - - operations = [ - migrations.AlterField( - model_name='independentmoney', - name='reporting_period', - field=models.ForeignKey(verbose_name='Form & Reporting Period', to='finance.ReportingPeriod', help_text='Form + date range'), - ), - ] diff --git a/finance/models.py b/finance/models.py index e8c3474d..3b845903 100644 --- a/finance/models.py +++ b/finance/models.py @@ -225,31 +225,6 @@ class Meta: verbose_name_plural = 'beneficiaries' ordering = Committee._meta.ordering - -@python_2_unicode_compatible -class ReportingPeriod(models.Model): - """Model tracking form reporting periods.""" - period_start = models.DateField() - period_end = models.DateField() - filing_deadline = models.DateField(blank=True, null=True, default=None) - form = models.ForeignKey('Form') - locality = models.ForeignKey( - 'locality.Locality', blank=True, null=True, default=None) - permanent = models.BooleanField( - default=True, help_text="Whether data is reported once, or re-reported.") - - def __str__(self): - locality_info = (' (%s)' % self.locality) if self.locality else '' - period_start_info = self.period_start or '' - period_end_info = self.period_end or '' - deadline_info = self.filing_deadline or '' - return '%s%s: %s to %s (due: %s)' % ( - self.form, locality_info, period_start_info, period_end_info, deadline_info) - - class Meta: - ordering = ('locality', 'period_start', 'period_end', 'form') - - @python_2_unicode_compatible class IndependentMoney(models.Model): """ @@ -261,10 +236,6 @@ class IndependentMoney(models.Model): cumulative_amount = models.FloatField( help_text="Total monetary value of provided benefits, to date of this transaction.", blank=True, null=True, default=None) - reporting_period = models.ForeignKey( - 'ReportingPeriod', - help_text="Form + date range", - verbose_name="Form & Reporting Period") report_date = models.DateField() benefactor_zip = models.ForeignKey('locality.ZipCode') @@ -289,13 +260,9 @@ def __str__(self): val += " in %s [%s]" % ( 'support of' if self.beneficiary.support else 'opposition to', self.beneficiary.ballot_item_selection) - val += ", reported via %s on %s" % ( - self.reporting_period.form, self.report_date) return val class Meta: verbose_name_plural = 'independent money' - ordering = ('-reporting_period__period_start', - '-reporting_period__period_end', - '-beneficiary__ballot_item_selection__ballot_item__ballot__date', # noqa + ordering = ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', # noqa '-report_date', ) diff --git a/finance/tests/test_strings.py b/finance/tests/test_strings.py index 7d7f635b..058b10c0 100644 --- a/finance/tests/test_strings.py +++ b/finance/tests/test_strings.py @@ -3,7 +3,7 @@ from .test_xformnetfilerawdata import WithForm460ADataTest from finance.models import (IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, PartyBenefactor, - Benefactor, Form, Committee, ReportingPeriod) + Benefactor, Form, Committee) class IndependentMoneyStringTests(WithForm460ADataTest, TestCase): @@ -20,7 +20,7 @@ def test_strings(self): for cls in [IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, Benefactor, - PartyBenefactor, Form, Committee, ReportingPeriod]: + PartyBenefactor, Form, Committee]: if cls.objects.all().count() == 0: # bad :( try: obj = cls() From ebc16035b70c6106bef5753013a896528fd119b2 Mon Sep 17 00:00:00 2001 From: Mike Ubell Date: Tue, 19 Jul 2016 20:53:22 -0700 Subject: [PATCH 02/14] Remove reporting_period from xformat #252 --- .../commands/xformnetfilerawdata.py | 104 ++++-------------- 1 file changed, 24 insertions(+), 80 deletions(-) diff --git a/finance/management/commands/xformnetfilerawdata.py b/finance/management/commands/xformnetfilerawdata.py index d9bbf84b..0709d708 100644 --- a/finance/management/commands/xformnetfilerawdata.py +++ b/finance/management/commands/xformnetfilerawdata.py @@ -161,36 +161,6 @@ def parse_party_from_name(committee_name): return Party.objects.get_or_create(name='Unknown')[0] -def parse_form_and_report_period(row, form, locality, verbosity=1): - # Create/get the relevant form objects. - form_name = form['form_name'] - form_type = form['form_type'] - if len(form_type) == 1: - form_type = '460' + form_type - - form, _ = models.Form.objects.get_or_create( - name=form_name, text_id=form_type) - - report_date = date_parse(row['tran_Date']) - reporting_periods = models.ReportingPeriod.objects.filter( - form=form, locality=locality, - period_start__lte=report_date, period_end__gte=report_date) - - if len(reporting_periods) == 0: - raise Exception(("You must add a reporting period for form '%s', " - "locality '%s', containing date %s") % ( - form, locality, report_date)) - - elif len(reporting_periods) > 1: - raise Exception("More than one reporting period matches; please " - "review.") - - # datetime.datetime(report_date.year, 1, 1), - # period_end=datetime.datetime(report_date.year, 12, 31)) - - return reporting_periods[0] - - def parse_beneficiary(row, agency, verbosity=1): # Parse and save the beneficiary, contribution. assert agency is not None, "Agency should be set." @@ -356,7 +326,7 @@ def clean_filer_id(filer_id): @transaction.atomic -def load_form_row(row, agency, reporting_period, force=False, verbosity=1): # noqa +def load_form_row(row, agency, force=False, verbosity=1): # noqa """ Loads an individual row from Form 460 Schedule A. # noqa This is where most of the magic happens! @@ -407,7 +377,7 @@ def load_form_row(row, agency, reporting_period, force=False, verbosity=1): # n "%s != %s" % (money.report_date, date_parse(row['tran_Date'])) if verbosity: print("Skipping existing [%s] row, %s/%s" % ( - reporting_period.form.name, money.source, money.source_xact_id)) + money.source, money.source_xact_id)) return money, False # Did not load. @@ -423,27 +393,33 @@ def load_form_row(row, agency, reporting_period, force=False, verbosity=1): # n beneficiary.save() # Now we have all the parts. Create and save it. - money = models.IndependentMoney( - source='NF', - source_xact_id=row['netFileKey'], - filing_id=row.get('filingId'), - amount=float(row['tran_Amt1']), - cumulative_amount=float(row.get('tran_Amt2', 0)) or None, - report_date=date_parse(row['tran_Date']), - reporting_period=reporting_period, - benefactor_zip=bf_zip_code, - benefactor=benefactor, - beneficiary=beneficiary) - money.save() + try: + money, created = models.IndependentMoney.objects.get_or_create( + source='NF', + source_xact_id=row['netFileKey'], + filing_id=row.get('filingId'), + amount=float(row['tran_Amt1']), + cumulative_amount=float(row.get('tran_Amt2', 0)) or None, + report_date=date_parse(row['tran_Date']), + benefactor_zip=bf_zip_code, + benefactor=benefactor, + beneficiary=beneficiary) + except Exception as E: + print str(E) + print row + raise E + if verbosity: + if created: + print "created" print(str(money)) return money, True @transaction.atomic -def load_form460d_row(row, agency, reporting_period, force=False, verbosity=1): # noqa +def load_form460d_row(row, agency, force=False, verbosity=1): # noqa """ Loads an individual row from Form 460 Schedule A. # noqa This is where most of the magic happens! @@ -479,8 +455,7 @@ def load_form460d_row(row, agency, reporting_period, force=False, verbosity=1): """ money, loaded = load_form_row( - row=row, agency=agency, reporting_period=reporting_period, - force=force, verbosity=verbosity) + row=row, agency=agency, force=force, verbosity=verbosity) # We can load some info about support/oppose. if loaded: @@ -531,17 +506,6 @@ def find_unloaded_rows(data, skip_rate=100, force=False, verbosity=1): else: yield set(xacts) - set(vals) # missing set - -def delete_form_data(reporting_period, verbosity=1): - rows_to_delete = models.IndependentMoney.objects.filter( - reporting_period=reporting_period) - if verbosity > 0: - print("Deleting %d rows of irrelevant %s data from between %s and %s." % ( - rows_to_delete.count(), reporting_period.form, - reporting_period.period_start, reporting_period.period_end)) - rows_to_delete.delete() - - def load_form_data(data, agency_fn, form_name, form_type=None, force=False, verbosity=1): """ @@ -553,7 +517,6 @@ def load_form_data(data, agency_fn, form_name, form_type=None, print("Attempting to load %d rows of %s data." % (len(data), form_name)) # Parse out the contributor information. - reporting_period = None # Set on first valid row. error_rows = [] xact_key_generator = find_unloaded_rows(data, force=force, verbosity=verbosity) xact_keys = [] @@ -573,33 +536,14 @@ def load_form_data(data, agency_fn, form_name, form_type=None, try: agency = agency_fn(minimal_row) - if reporting_period is None: - # Query form period. - form = dict(form_name=form_name, form_type=form_type) - locality = City.objects.get(short_name=agency['shortcut']) - reporting_period = parse_form_and_report_period( - minimal_row, form, locality=locality, verbosity=verbosity) - - # Verify that the data from this form are relevant. - now = datetime.datetime.date(datetime.datetime.now()) - time_to_skip = (now < reporting_period.period_start or - now > reporting_period.period_end) - - if not reporting_period.permanent and time_to_skip: - # No longer relevant. Delete any existing data, and skip this. - delete_form_data(reporting_period) - raise SkipForm() - # 460D data is different... if form_type == 'D': minimal_row['entity_Cd'] = minimal_row.get('entity_Cd', 'CRT') load_form460d_row( - minimal_row, agency=agency, reporting_period=reporting_period, - force=force, verbosity=verbosity) + minimal_row, agency=agency, force=force, verbosity=verbosity) else: load_form_row( - minimal_row, agency=agency, reporting_period=reporting_period, - force=force, verbosity=verbosity) + minimal_row, agency=agency, force=force, verbosity=verbosity) except Exception as ex: error_rows.append((ri, raw_row, minimal_row, ex)) # TODO: Store errors, for review later. From 26c9d10c75c3ad90be521fa2fd40167e61e1dd33 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Tue, 19 Jul 2016 20:12:46 -0700 Subject: [PATCH 03/14] Remove ReportingPeriod models and fixtures --- finance/admin.py | 3 +- finance/management/commands/resolve_money.py | 2 +- finance/migrations/0001_initial.py | 19 +--------- finance/migrations/0004_auto_20160227_1512.py | 19 ---------- finance/models.py | 35 +------------------ finance/tests/test_strings.py | 4 +-- 6 files changed, 6 insertions(+), 76 deletions(-) delete mode 100644 finance/migrations/0004_auto_20160227_1512.py diff --git a/finance/admin.py b/finance/admin.py index 14735b76..4ec03c7d 100644 --- a/finance/admin.py +++ b/finance/admin.py @@ -32,12 +32,11 @@ class BeneficiaryAdmin(admin.ModelAdmin): class IndependentMoneyAdmin(admin.ModelAdmin): readonly_fields = ('benefactor', 'benefactor_zip', 'beneficiary', 'amount', 'cumulative_amount', 'report_date', - 'reporting_period', 'source', 'source_xact_id') + 'source', 'source_xact_id') admin.site.register(models.Committee) admin.site.register(models.Employer) admin.site.register(models.Form) -admin.site.register(models.ReportingPeriod) validate_and_register_admin( models.Beneficiary, BeneficiaryAdmin, num_hidden_fields=5) diff --git a/finance/management/commands/resolve_money.py b/finance/management/commands/resolve_money.py index 33d99d30..201aa5ef 100644 --- a/finance/management/commands/resolve_money.py +++ b/finance/management/commands/resolve_money.py @@ -22,5 +22,5 @@ def handle(self, *args, **options): print related_money.index(im), str(im) for rm in related_money: print "\t%s %10s %s %s %s" % ( - rm.report_date, str(rm.reporting_period)[:10], rm.filing_id, + rm.report_date, rm.filing_id, rm.source_xact_id, rm.benefactor) diff --git a/finance/migrations/0001_initial.py b/finance/migrations/0001_initial.py index c171dde2..0515f94f 100644 --- a/finance/migrations/0001_initial.py +++ b/finance/migrations/0001_initial.py @@ -65,22 +65,10 @@ class Migration(migrations.Migration): ('source_xact_id', models.CharField(help_text='Transaction ID (specific to data source)', max_length=32)), ], options={ - 'ordering': ('-reporting_period__period_start', '-reporting_period__period_end', '-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), + 'ordering': ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), 'verbose_name_plural': 'independent money', }, ), - migrations.CreateModel( - name='ReportingPeriod', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('period_start', models.DateField()), - ('period_end', models.DateField()), - ('form', models.ForeignKey(to='finance.Form')), - ], - options={ - 'ordering': ('period_start', 'period_end'), - }, - ), migrations.CreateModel( name='Beneficiary', fields=[ @@ -169,11 +157,6 @@ class Migration(migrations.Migration): name='benefactor_zip', field=models.ForeignKey(to='locality.ZipCode'), ), - migrations.AddField( - model_name='independentmoney', - name='reporting_period', - field=models.ForeignKey(to='finance.ReportingPeriod'), - ), migrations.AddField( model_name='committee', name='city', diff --git a/finance/migrations/0004_auto_20160227_1512.py b/finance/migrations/0004_auto_20160227_1512.py deleted file mode 100644 index 645209de..00000000 --- a/finance/migrations/0004_auto_20160227_1512.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('finance', '0003_merge'), - ] - - operations = [ - migrations.AlterField( - model_name='independentmoney', - name='reporting_period', - field=models.ForeignKey(verbose_name='Form & Reporting Period', to='finance.ReportingPeriod', help_text='Form + date range'), - ), - ] diff --git a/finance/models.py b/finance/models.py index e8c3474d..3b845903 100644 --- a/finance/models.py +++ b/finance/models.py @@ -225,31 +225,6 @@ class Meta: verbose_name_plural = 'beneficiaries' ordering = Committee._meta.ordering - -@python_2_unicode_compatible -class ReportingPeriod(models.Model): - """Model tracking form reporting periods.""" - period_start = models.DateField() - period_end = models.DateField() - filing_deadline = models.DateField(blank=True, null=True, default=None) - form = models.ForeignKey('Form') - locality = models.ForeignKey( - 'locality.Locality', blank=True, null=True, default=None) - permanent = models.BooleanField( - default=True, help_text="Whether data is reported once, or re-reported.") - - def __str__(self): - locality_info = (' (%s)' % self.locality) if self.locality else '' - period_start_info = self.period_start or '' - period_end_info = self.period_end or '' - deadline_info = self.filing_deadline or '' - return '%s%s: %s to %s (due: %s)' % ( - self.form, locality_info, period_start_info, period_end_info, deadline_info) - - class Meta: - ordering = ('locality', 'period_start', 'period_end', 'form') - - @python_2_unicode_compatible class IndependentMoney(models.Model): """ @@ -261,10 +236,6 @@ class IndependentMoney(models.Model): cumulative_amount = models.FloatField( help_text="Total monetary value of provided benefits, to date of this transaction.", blank=True, null=True, default=None) - reporting_period = models.ForeignKey( - 'ReportingPeriod', - help_text="Form + date range", - verbose_name="Form & Reporting Period") report_date = models.DateField() benefactor_zip = models.ForeignKey('locality.ZipCode') @@ -289,13 +260,9 @@ def __str__(self): val += " in %s [%s]" % ( 'support of' if self.beneficiary.support else 'opposition to', self.beneficiary.ballot_item_selection) - val += ", reported via %s on %s" % ( - self.reporting_period.form, self.report_date) return val class Meta: verbose_name_plural = 'independent money' - ordering = ('-reporting_period__period_start', - '-reporting_period__period_end', - '-beneficiary__ballot_item_selection__ballot_item__ballot__date', # noqa + ordering = ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', # noqa '-report_date', ) diff --git a/finance/tests/test_strings.py b/finance/tests/test_strings.py index 7d7f635b..058b10c0 100644 --- a/finance/tests/test_strings.py +++ b/finance/tests/test_strings.py @@ -3,7 +3,7 @@ from .test_xformnetfilerawdata import WithForm460ADataTest from finance.models import (IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, PartyBenefactor, - Benefactor, Form, Committee, ReportingPeriod) + Benefactor, Form, Committee) class IndependentMoneyStringTests(WithForm460ADataTest, TestCase): @@ -20,7 +20,7 @@ def test_strings(self): for cls in [IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, Benefactor, - PartyBenefactor, Form, Committee, ReportingPeriod]: + PartyBenefactor, Form, Committee]: if cls.objects.all().count() == 0: # bad :( try: obj = cls() From ea492341ad4d44bafebfd2ad2934a9845c168bdf Mon Sep 17 00:00:00 2001 From: Mike Ubell Date: Tue, 19 Jul 2016 21:14:46 -0700 Subject: [PATCH 04/14] fix migrations --- .../0005_independentmoney_filing_id.py | 2 +- finance/migrations/0006_auto_20160320_1956.py | 18 +----------------- .../0007_reportingperiod_filing_deadline.py | 13 +------------ finance/migrations/0008_auto_20160320_2349.py | 5 ----- 4 files changed, 3 insertions(+), 35 deletions(-) diff --git a/finance/migrations/0005_independentmoney_filing_id.py b/finance/migrations/0005_independentmoney_filing_id.py index 930182ad..448cc8bc 100644 --- a/finance/migrations/0005_independentmoney_filing_id.py +++ b/finance/migrations/0005_independentmoney_filing_id.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('finance', '0004_auto_20160227_1512'), + ('finance', '0003_merge'), ] operations = [ diff --git a/finance/migrations/0006_auto_20160320_1956.py b/finance/migrations/0006_auto_20160320_1956.py index 92ccab87..2d859b70 100644 --- a/finance/migrations/0006_auto_20160320_1956.py +++ b/finance/migrations/0006_auto_20160320_1956.py @@ -5,20 +5,4 @@ class Migration(migrations.Migration): - - dependencies = [ - ('locality', '0002_locality_true_model_id'), - ('finance', '0005_independentmoney_filing_id'), - ] - - operations = [ - migrations.AlterModelOptions( - name='reportingperiod', - options={'ordering': ('locality', 'period_start', 'period_end', 'form')}, - ), - migrations.AddField( - model_name='reportingperiod', - name='locality', - field=models.ForeignKey(default=None, blank=True, to='locality.Locality', null=True), - ), - ] + pass diff --git a/finance/migrations/0007_reportingperiod_filing_deadline.py b/finance/migrations/0007_reportingperiod_filing_deadline.py index 5648454c..2d859b70 100644 --- a/finance/migrations/0007_reportingperiod_filing_deadline.py +++ b/finance/migrations/0007_reportingperiod_filing_deadline.py @@ -5,15 +5,4 @@ class Migration(migrations.Migration): - - dependencies = [ - ('finance', '0006_auto_20160320_1956'), - ] - - operations = [ - migrations.AddField( - model_name='reportingperiod', - name='filing_deadline', - field=models.DateField(default=None, null=True, blank=True), - ), - ] + pass diff --git a/finance/migrations/0008_auto_20160320_2349.py b/finance/migrations/0008_auto_20160320_2349.py index ce278a43..1e5d6385 100644 --- a/finance/migrations/0008_auto_20160320_2349.py +++ b/finance/migrations/0008_auto_20160320_2349.py @@ -11,11 +11,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='reportingperiod', - name='permanent', - field=models.BooleanField(default=True), - ), migrations.AlterField( model_name='form', name='text_id', From e4e5aa9996f609ad2398106beb5ff2f9070d73e5 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Wed, 20 Jul 2016 00:27:45 -0700 Subject: [PATCH 05/14] Restore old migrations (cuz I wasnt supposed to delete them) --- finance/migrations/0001_initial.py | 19 ++++++++++++++++++- finance/migrations/0004_auto_20160227_1512.py | 19 +++++++++++++++++++ .../0005_independentmoney_filing_id.py | 2 +- finance/migrations/0006_auto_20160320_1956.py | 18 +++++++++++++++++- .../0007_reportingperiod_filing_deadline.py | 13 ++++++++++++- finance/migrations/0008_auto_20160320_2349.py | 5 +++++ 6 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 finance/migrations/0004_auto_20160227_1512.py diff --git a/finance/migrations/0001_initial.py b/finance/migrations/0001_initial.py index 0515f94f..c171dde2 100644 --- a/finance/migrations/0001_initial.py +++ b/finance/migrations/0001_initial.py @@ -65,10 +65,22 @@ class Migration(migrations.Migration): ('source_xact_id', models.CharField(help_text='Transaction ID (specific to data source)', max_length=32)), ], options={ - 'ordering': ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), + 'ordering': ('-reporting_period__period_start', '-reporting_period__period_end', '-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), 'verbose_name_plural': 'independent money', }, ), + migrations.CreateModel( + name='ReportingPeriod', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('period_start', models.DateField()), + ('period_end', models.DateField()), + ('form', models.ForeignKey(to='finance.Form')), + ], + options={ + 'ordering': ('period_start', 'period_end'), + }, + ), migrations.CreateModel( name='Beneficiary', fields=[ @@ -157,6 +169,11 @@ class Migration(migrations.Migration): name='benefactor_zip', field=models.ForeignKey(to='locality.ZipCode'), ), + migrations.AddField( + model_name='independentmoney', + name='reporting_period', + field=models.ForeignKey(to='finance.ReportingPeriod'), + ), migrations.AddField( model_name='committee', name='city', diff --git a/finance/migrations/0004_auto_20160227_1512.py b/finance/migrations/0004_auto_20160227_1512.py new file mode 100644 index 00000000..645209de --- /dev/null +++ b/finance/migrations/0004_auto_20160227_1512.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('finance', '0003_merge'), + ] + + operations = [ + migrations.AlterField( + model_name='independentmoney', + name='reporting_period', + field=models.ForeignKey(verbose_name='Form & Reporting Period', to='finance.ReportingPeriod', help_text='Form + date range'), + ), + ] diff --git a/finance/migrations/0005_independentmoney_filing_id.py b/finance/migrations/0005_independentmoney_filing_id.py index 448cc8bc..930182ad 100644 --- a/finance/migrations/0005_independentmoney_filing_id.py +++ b/finance/migrations/0005_independentmoney_filing_id.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('finance', '0003_merge'), + ('finance', '0004_auto_20160227_1512'), ] operations = [ diff --git a/finance/migrations/0006_auto_20160320_1956.py b/finance/migrations/0006_auto_20160320_1956.py index 2d859b70..92ccab87 100644 --- a/finance/migrations/0006_auto_20160320_1956.py +++ b/finance/migrations/0006_auto_20160320_1956.py @@ -5,4 +5,20 @@ class Migration(migrations.Migration): - pass + + dependencies = [ + ('locality', '0002_locality_true_model_id'), + ('finance', '0005_independentmoney_filing_id'), + ] + + operations = [ + migrations.AlterModelOptions( + name='reportingperiod', + options={'ordering': ('locality', 'period_start', 'period_end', 'form')}, + ), + migrations.AddField( + model_name='reportingperiod', + name='locality', + field=models.ForeignKey(default=None, blank=True, to='locality.Locality', null=True), + ), + ] diff --git a/finance/migrations/0007_reportingperiod_filing_deadline.py b/finance/migrations/0007_reportingperiod_filing_deadline.py index 2d859b70..5648454c 100644 --- a/finance/migrations/0007_reportingperiod_filing_deadline.py +++ b/finance/migrations/0007_reportingperiod_filing_deadline.py @@ -5,4 +5,15 @@ class Migration(migrations.Migration): - pass + + dependencies = [ + ('finance', '0006_auto_20160320_1956'), + ] + + operations = [ + migrations.AddField( + model_name='reportingperiod', + name='filing_deadline', + field=models.DateField(default=None, null=True, blank=True), + ), + ] diff --git a/finance/migrations/0008_auto_20160320_2349.py b/finance/migrations/0008_auto_20160320_2349.py index 1e5d6385..ce278a43 100644 --- a/finance/migrations/0008_auto_20160320_2349.py +++ b/finance/migrations/0008_auto_20160320_2349.py @@ -11,6 +11,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='reportingperiod', + name='permanent', + field=models.BooleanField(default=True), + ), migrations.AlterField( model_name='form', name='text_id', From 6089ca8c72dad23f4d88b0ba2067d981e68908cc Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Wed, 20 Jul 2016 00:28:04 -0700 Subject: [PATCH 06/14] Add migration to delete reporting_period model --- finance/migrations/0009_auto_20160720_0725.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 finance/migrations/0009_auto_20160720_0725.py diff --git a/finance/migrations/0009_auto_20160720_0725.py b/finance/migrations/0009_auto_20160720_0725.py new file mode 100644 index 00000000..0ca74d34 --- /dev/null +++ b/finance/migrations/0009_auto_20160720_0725.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('finance', '0008_auto_20160320_2349'), + ] + + operations = [ + migrations.RemoveField( + model_name='reportingperiod', + name='form', + ), + migrations.RemoveField( + model_name='reportingperiod', + name='locality', + ), + migrations.AlterModelOptions( + name='independentmoney', + options={'ordering': ('-beneficiary__ballot_item_selection__ballot_item__ballot__date', '-report_date'), 'verbose_name_plural': 'independent money'}, + ), + migrations.RemoveField( + model_name='independentmoney', + name='reporting_period', + ), + migrations.DeleteModel( + name='ReportingPeriod', + ), + ] From 1bd16cff4c5fd31cc8874b0c670cef879915fbff Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Wed, 20 Jul 2016 10:44:00 -0700 Subject: [PATCH 07/14] Protected get to tran_NamL --- finance/management/commands/xformnetfilerawdata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/finance/management/commands/xformnetfilerawdata.py b/finance/management/commands/xformnetfilerawdata.py index 0709d708..6906c28d 100644 --- a/finance/management/commands/xformnetfilerawdata.py +++ b/finance/management/commands/xformnetfilerawdata.py @@ -102,7 +102,7 @@ def parse_benefactor(row, verbosity=1): middle_name = raw_name[len(first_name):].strip() benefactor, _ = models.PersonBenefactor.objects.get_or_create( first_name=first_name, middle_name=middle_name, - last_name=clean_name(row['tran_NamL']), + last_name=clean_name(row.get('tran_NamL')) or '', employer=employer, city=bf_city, state=bf_state, @@ -113,7 +113,7 @@ def parse_benefactor(row, verbosity=1): elif row['entity_Cd'] == 'OTH': # Commerial benefactor or Other benefactor, _ = models.OtherBenefactor.objects \ - .get_or_create(name=clean_name(row['tran_NamL'])) + .get_or_create(name=clean_name(row.get('tran_NamL')) or '') benefactor.benefactor_locality = bf_city benefactor.save() @@ -135,7 +135,7 @@ def parse_benefactor(row, verbosity=1): assert queryset.count() == 1, "Avoid duplicate committees" elif row['entity_Cd'] in ['PTY']: - name = clean_name(row['tran_NamL']) + name = clean_name(row.get('tran_NamL')) or '' party = parse_party_from_name(name) benefactor, _ = models.PartyBenefactor.objects \ .get_or_create(name=name, party=party) From b901e06a51f4609571e589b8f5024a2baf7c1f17 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Wed, 20 Jul 2016 15:11:42 -0700 Subject: [PATCH 08/14] Fix linting errors --- finance/management/commands/xformnetfilerawdata.py | 3 +-- finance/models.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/finance/management/commands/xformnetfilerawdata.py b/finance/management/commands/xformnetfilerawdata.py index 6906c28d..7141844f 100644 --- a/finance/management/commands/xformnetfilerawdata.py +++ b/finance/management/commands/xformnetfilerawdata.py @@ -4,7 +4,6 @@ See https://netfile.com/Filer/Content/docs/cal_format_201.pdf for documentation. """ -import datetime import warnings from dateutil.parser import parse as date_parse from itertools import izip_longest @@ -409,7 +408,6 @@ def load_form_row(row, agency, force=False, verbosity=1): # noqa print row raise E - if verbosity: if created: print "created" @@ -506,6 +504,7 @@ def find_unloaded_rows(data, skip_rate=100, force=False, verbosity=1): else: yield set(xacts) - set(vals) # missing set + def load_form_data(data, agency_fn, form_name, form_type=None, force=False, verbosity=1): """ diff --git a/finance/models.py b/finance/models.py index 3b845903..2d1d8937 100644 --- a/finance/models.py +++ b/finance/models.py @@ -225,6 +225,7 @@ class Meta: verbose_name_plural = 'beneficiaries' ordering = Committee._meta.ordering + @python_2_unicode_compatible class IndependentMoney(models.Model): """ From 1c76760d7280802bac358521b28c1e0546ecc17d Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sun, 24 Jul 2016 22:01:34 -0700 Subject: [PATCH 09/14] Fix test by clearing Pary objects before asserting --- finance/tests/test_xformnetfilerawdata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/finance/tests/test_xformnetfilerawdata.py b/finance/tests/test_xformnetfilerawdata.py index 277ee4fb..2aecca0a 100644 --- a/finance/tests/test_xformnetfilerawdata.py +++ b/finance/tests/test_xformnetfilerawdata.py @@ -120,6 +120,10 @@ class XformNetfileRawDataPTYTest(TestCase): @classmethod def setUpClass(cls): TestCase.setUpClass() + + # A count of Party objects is used in test assertion + Party.objects.all().delete() + cls.PTY_CSV_FILE = op.join( op.dirname(__file__), 'data', 'test_PTY.csv') From 14b37439a14d0ae914c7d6fcfd4722f9f549c9e4 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sun, 24 Jul 2016 22:06:57 -0700 Subject: [PATCH 10/14] Fix linting --- ballot/tests/test_api_ballot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ballot/tests/test_api_ballot.py b/ballot/tests/test_api_ballot.py index c3d30b30..080c73ae 100644 --- a/ballot/tests/test_api_ballot.py +++ b/ballot/tests/test_api_ballot.py @@ -42,7 +42,6 @@ def test_current_ballot(self): ballot_url = reverse( 'current_ballot', kwargs={'locality_id': self.ballot.locality_id}) resp = self.client.get(ballot_url) - self.assertEqual(self.ballot.id, resp.data['id']) self.assertEqual(self.ballot.date, resp.data['date']) self.assertEqual(self.ballot.locality.id, resp.data['locality_id']) From 644016c55d4994e12ab036e8f5ac82572bf86005 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sun, 24 Jul 2016 22:10:42 -0700 Subject: [PATCH 11/14] Remove Form model --- finance/admin.py | 1 - finance/migrations/0010_auto_20160725_0504.py | 21 +++++++++++++++ finance/models.py | 27 ------------------- finance/tests/test_strings.py | 4 +-- 4 files changed, 23 insertions(+), 30 deletions(-) create mode 100644 finance/migrations/0010_auto_20160725_0504.py diff --git a/finance/admin.py b/finance/admin.py index 4ec03c7d..f80055ec 100644 --- a/finance/admin.py +++ b/finance/admin.py @@ -36,7 +36,6 @@ class IndependentMoneyAdmin(admin.ModelAdmin): admin.site.register(models.Committee) admin.site.register(models.Employer) -admin.site.register(models.Form) validate_and_register_admin( models.Beneficiary, BeneficiaryAdmin, num_hidden_fields=5) diff --git a/finance/migrations/0010_auto_20160725_0504.py b/finance/migrations/0010_auto_20160725_0504.py new file mode 100644 index 00000000..784f1ea4 --- /dev/null +++ b/finance/migrations/0010_auto_20160725_0504.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('finance', '0009_auto_20160720_0725'), + ] + + operations = [ + migrations.RemoveField( + model_name='form', + name='locality', + ), + migrations.DeleteModel( + name='Form', + ), + ] diff --git a/finance/models.py b/finance/models.py index 2d1d8937..f66539dd 100644 --- a/finance/models.py +++ b/finance/models.py @@ -69,33 +69,6 @@ class Meta: ordering = ('name', 'locality__name', 'locality__short_name') -@python_2_unicode_compatible -class Form(models.Model): - """ - Information about finance reporting forms. - """ - FREQUENCY_TYPES = ( - ('24', '24 hours'), - ('SA', 'Semi-annual'), - ('QU', 'Quarterly'), - ('OT', 'Other') - ) - - name = models.CharField(max_length=255) - text_id = models.CharField(max_length=32, help_text='e.g. 460A') - submission_frequency = models.CharField( - max_length=2, choices=FREQUENCY_TYPES) - locality = models.ForeignKey('locality.Locality', blank=True, null=True, default=None, - help_text="Only set when a form is specific " - "to a locality.") - - def __str__(self): - return self.name - - class Meta: - ordering = ('locality__name', 'locality__short_name', 'name') - - @python_2_unicode_compatible class Benefactor(models.Model, ReverseLookupStringMixin): """ diff --git a/finance/tests/test_strings.py b/finance/tests/test_strings.py index 058b10c0..93804871 100644 --- a/finance/tests/test_strings.py +++ b/finance/tests/test_strings.py @@ -3,7 +3,7 @@ from .test_xformnetfilerawdata import WithForm460ADataTest from finance.models import (IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, PartyBenefactor, - Benefactor, Form, Committee) + Benefactor, Committee) class IndependentMoneyStringTests(WithForm460ADataTest, TestCase): @@ -20,7 +20,7 @@ def test_strings(self): for cls in [IndependentMoney, Beneficiary, CommitteeBenefactor, OtherBenefactor, PersonBenefactor, Benefactor, - PartyBenefactor, Form, Committee]: + PartyBenefactor, Committee]: if cls.objects.all().count() == 0: # bad :( try: obj = cls() From 65d202bfb739e34da9abf1d0745853ac4abadd9c Mon Sep 17 00:00:00 2001 From: Mike Ubell Date: Sun, 14 Aug 2016 12:55:11 -0700 Subject: [PATCH 12/14] Add missing fields. Logging vs printing --- .../commands/xformnetfilerawdata.py | 19 +++++++---- .../migrations/0002_auto_20160810_0335.py | 34 +++++++++++++++++++ netfile_raw/models.py | 14 ++++++++ 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 netfile_raw/migrations/0002_auto_20160810_0335.py diff --git a/finance/management/commands/xformnetfilerawdata.py b/finance/management/commands/xformnetfilerawdata.py index 7141844f..431de7d2 100644 --- a/finance/management/commands/xformnetfilerawdata.py +++ b/finance/management/commands/xformnetfilerawdata.py @@ -5,6 +5,7 @@ """ import warnings +import logging from dateutil.parser import parse as date_parse from itertools import izip_longest from numbers import Number @@ -408,10 +409,10 @@ def load_form_row(row, agency, force=False, verbosity=1): # noqa print row raise E - if verbosity: - if created: - print "created" - print(str(money)) + c = "" + if created: + c = "created " + logging.debug("%s%s", c, str(money)) return money, True @@ -495,9 +496,8 @@ def find_unloaded_rows(data, skip_rate=100, force=False, verbosity=1): for xacts in grouper(skip_rate, xact_keys): vals = [v['source_xact_id'] for v in models.IndependentMoney.objects.filter( source='NF', source_xact_id__in=xacts).values('source_xact_id')] - if verbosity: - for val in vals: - print("Skipping NF/%s" % val) + for val in vals: + logging.debug("Skipping NF/%s", val) if force: yield set(xacts) @@ -519,6 +519,7 @@ def load_form_data(data, agency_fn, form_name, form_type=None, error_rows = [] xact_key_generator = find_unloaded_rows(data, force=force, verbosity=verbosity) xact_keys = [] + count = 0; for ri, (_, raw_row) in enumerate(data.T.iteritems()): # Quickly get near an unloaded row. while not xact_keys and xact_keys is not None: @@ -548,6 +549,10 @@ def load_form_data(data, agency_fn, form_name, form_type=None, # TODO: Store errors, for review later. raise + count += 1 + if (count % 1000) == 0: + print("Loaded %d records" % count) + return error_rows diff --git a/netfile_raw/migrations/0002_auto_20160810_0335.py b/netfile_raw/migrations/0002_auto_20160810_0335.py new file mode 100644 index 00000000..56735b34 --- /dev/null +++ b/netfile_raw/migrations/0002_auto_20160810_0335.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('netfile_raw', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='netfilecal201transaction', + name='filerLocalId', + field=models.CharField(max_length=32, null=True, db_column='filerLocalId'), + ), + migrations.AddField( + model_name='netfilecal201transaction', + name='filerStateId', + field=models.CharField(max_length=2, null=True, db_column='filerStateId'), + ), + migrations.AddField( + model_name='netfilecal201transaction', + name='filingEndDate', + field=models.DateField(null=True, db_column='filingEndDate'), + ), + migrations.AddField( + model_name='netfilecal201transaction', + name='filingStartDate', + field=models.DateField(null=True, db_column='filingStartDate'), + ), + ] diff --git a/netfile_raw/models.py b/netfile_raw/models.py index 2eef63c8..eb7ea987 100644 --- a/netfile_raw/models.py +++ b/netfile_raw/models.py @@ -75,6 +75,9 @@ class NetFileCal201Transaction(CalAccessBaseModel): tran_NamT = models.CharField( max_length=256, db_column='tran_NamT') + filingStartDate = models.DateField( + null=True, + db_column='filingStartDate') tran_Dscr = models.CharField( max_length=256, db_column='tran_Dscr') @@ -150,6 +153,9 @@ class NetFileCal201Transaction(CalAccessBaseModel): intr_NamF = models.CharField( max_length=256, db_column='intr_NamF') + filingEndDate = models.DateField( + null=True, + db_column='filingEndDate') transactionType = models.IntegerField( null=True, db_column='transactionType') @@ -255,6 +261,10 @@ class NetFileCal201Transaction(CalAccessBaseModel): int_CmteId = models.CharField( max_length=32, db_column='int_CmteId') + filerLocalId = models.CharField( + null=True, + max_length=32, + db_column='filerLocalId') calculated_Date = models.DateField( null=True, db_column='calculated_Date') @@ -288,6 +298,10 @@ class NetFileCal201Transaction(CalAccessBaseModel): entity_Cd = models.CharField( max_length=3, db_column='entity_Cd') + filerStateId = models.CharField( + max_length=2, + null=True, + db_column='filerStateId') tres_Adr1 = models.CharField( max_length=256, db_column='tres_Adr1') From 8c55bf049e8f9028964344c22060d6abaae510fd Mon Sep 17 00:00:00 2001 From: Mike Ubell Date: Sun, 14 Aug 2016 13:49:36 -0700 Subject: [PATCH 13/14] Remove merge --- finance/migrations/0009_auto_20160706_0339.py | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 finance/migrations/0009_auto_20160706_0339.py diff --git a/finance/migrations/0009_auto_20160706_0339.py b/finance/migrations/0009_auto_20160706_0339.py deleted file mode 100644 index 7ce7f492..00000000 --- a/finance/migrations/0009_auto_20160706_0339.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -import os.path as op - -from django.conf import settings -from django.core.management import call_command -from django.db import models, migrations - - -def load_ballot_data(apps, schema_editor): - fixtures = ( - ('finance', op.join(settings.FIXTURES_DIR, '2016_07_05_form.json')), - ('finance', op.join(settings.FIXTURES_DIR, '2016_07_05_reporting_period.json')), - ) - - for app, path in fixtures: - call_command('loaddata', path, app=app) - - -class Migration(migrations.Migration): - - dependencies = [ - ('finance', '0008_auto_20160320_2349'), - ] - - operations = [ - migrations.AlterField( - model_name='reportingperiod', - name='permanent', - field=models.BooleanField(default=True, help_text='Whether data is reported once, or re-reported.'), - ), - migrations.RunPython(load_ballot_data), - ] From 6d8190424bc161a26640e2964514e25482982e94 Mon Sep 17 00:00:00 2001 From: Mike Ubell Date: Mon, 15 Aug 2016 10:01:17 -0700 Subject: [PATCH 14/14] Update for review comments. Add missing merge file. --- disclosure/fixtures/2016_07_05_form.json | 42 -------- .../fixtures/2016_07_05_reporting_period.json | 98 ------------------- .../commands/xformnetfilerawdata.py | 6 +- .../migrations/0003_auto_20160815_1654.py | 19 ++++ 4 files changed, 22 insertions(+), 143 deletions(-) delete mode 100644 disclosure/fixtures/2016_07_05_form.json delete mode 100644 disclosure/fixtures/2016_07_05_reporting_period.json create mode 100644 netfile_raw/migrations/0003_auto_20160815_1654.py diff --git a/disclosure/fixtures/2016_07_05_form.json b/disclosure/fixtures/2016_07_05_form.json deleted file mode 100644 index b993b6b4..00000000 --- a/disclosure/fixtures/2016_07_05_form.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "fields": { - "locality": null, - "name": "Form 460 Schedule A", - "submission_frequency": "", - "text_id": "460A" - }, - "model": "finance.form", - "pk": 1 - }, - { - "fields": { - "locality": null, - "name": "Form 460 Schedule C", - "submission_frequency": "", - "text_id": "460C" - }, - "model": "finance.form", - "pk": 2 - }, - { - "fields": { - "locality": null, - "name": "Form 460 Schedule D", - "submission_frequency": "", - "text_id": "460D" - }, - "model": "finance.form", - "pk": 3 - }, - { - "fields": { - "locality": null, - "name": "Form 497", - "submission_frequency": "", - "text_id": "F497P1" - }, - "model": "finance.form", - "pk": 4 - } -] diff --git a/disclosure/fixtures/2016_07_05_reporting_period.json b/disclosure/fixtures/2016_07_05_reporting_period.json deleted file mode 100644 index a9bbf320..00000000 --- a/disclosure/fixtures/2016_07_05_reporting_period.json +++ /dev/null @@ -1,98 +0,0 @@ -[ - { - "fields": { - "filing_deadline": null, - "form": 1, - "locality": 2, - "period_end": "2016-12-31", - "period_start": "2016-01-01", - "permanent": false - }, - "model": "finance.reportingperiod", - "pk": 1 - }, - { - "fields": { - "filing_deadline": null, - "form": 2, - "locality": 2, - "period_end": "2016-12-31", - "period_start": "2016-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 2 - }, - { - "fields": { - "filing_deadline": null, - "form": 3, - "locality": 2, - "period_end": "2016-12-31", - "period_start": "2016-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 3 - }, - { - "fields": { - "filing_deadline": null, - "form": 4, - "locality": 2, - "period_end": "2016-12-31", - "period_start": "2016-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 4 - }, - { - "fields": { - "filing_deadline": null, - "form": 1, - "locality": 2, - "period_end": "2015-12-31", - "period_start": "2015-01-01", - "permanent": false - }, - "model": "finance.reportingperiod", - "pk": 5 - }, - { - "fields": { - "filing_deadline": null, - "form": 2, - "locality": 2, - "period_end": "2015-12-31", - "period_start": "2015-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 6 - }, - { - "fields": { - "filing_deadline": null, - "form": 3, - "locality": 2, - "period_end": "2015-12-31", - "period_start": "2015-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 7 - }, - { - "fields": { - "filing_deadline": null, - "form": 4, - "locality": 2, - "period_end": "2015-12-31", - "period_start": "2015-01-01", - "permanent": true - }, - "model": "finance.reportingperiod", - "pk": 8 - } -] diff --git a/finance/management/commands/xformnetfilerawdata.py b/finance/management/commands/xformnetfilerawdata.py index 5c44027d..ef6a965b 100644 --- a/finance/management/commands/xformnetfilerawdata.py +++ b/finance/management/commands/xformnetfilerawdata.py @@ -113,7 +113,7 @@ def parse_benefactor(row, verbosity=1): elif row['entity_Cd'] == 'OTH': # Commerial benefactor or Other benefactor, _ = models.OtherBenefactor.objects \ - .get_or_create(name=clean_name(row.get('tran_NamL')) or '') + .get_or_create(name=clean_name(row.get('tran_NamL', ''))) benefactor.benefactor_locality = bf_city benefactor.save() @@ -412,7 +412,7 @@ def load_form_row(row, agency, force=False, verbosity=1): # noqa c = "" if created: c = "created " - logging.debug("%s%s", c, str(money)) + logging.debug("%s%s", c, str(money)) return money, True @@ -519,7 +519,7 @@ def load_form_data(data, agency_fn, form_name, form_type=None, error_rows = [] xact_key_generator = find_unloaded_rows(data, force=force, verbosity=verbosity) xact_keys = [] - count = 0; + count = 0 for ri, (_, raw_row) in enumerate(data.T.iteritems()): # Quickly get near an unloaded row. while not xact_keys and xact_keys is not None: diff --git a/netfile_raw/migrations/0003_auto_20160815_1654.py b/netfile_raw/migrations/0003_auto_20160815_1654.py new file mode 100644 index 00000000..9c97d4fa --- /dev/null +++ b/netfile_raw/migrations/0003_auto_20160815_1654.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('netfile_raw', '0002_auto_20160810_0335'), + ] + + operations = [ + migrations.AlterField( + model_name='netfilecal201transaction', + name='filerStateId', + field=models.CharField(max_length=32, null=True, db_column='filerStateId'), + ), + ]