From b8f42410e4dd2dd100c00fb978dc4f138e375bba Mon Sep 17 00:00:00 2001
From: Tom Dooner <tom.dooner@brigade.com>
Date: Wed, 10 Aug 2016 20:23:58 -0700
Subject: [PATCH 1/2] Add detailed assertion fail message

---
 netfile_raw/management/commands/downloadnetfilerawdata.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/netfile_raw/management/commands/downloadnetfilerawdata.py b/netfile_raw/management/commands/downloadnetfilerawdata.py
index 62bf91cf..d1269171 100644
--- a/netfile_raw/management/commands/downloadnetfilerawdata.py
+++ b/netfile_raw/management/commands/downloadnetfilerawdata.py
@@ -212,7 +212,8 @@ def combine(self):
                         headers_written = headers
                     else:
                         # make sure things don't go all wierd between files.
-                        assert headers == headers_written
+                        assert headers == headers_written, \
+                            'Headers in %s do not match the first written csv' % (csv_path,)
 
                     for line in agency_csv.readlines():
                         combined_csv.write(','.join([agency, line]))

From 5fdb4b41895398dda4fa63197cc4aa7ee94676b5 Mon Sep 17 00:00:00 2001
From: Tom Dooner <tom.dooner@brigade.com>
Date: Sun, 14 Aug 2016 13:41:56 -0700
Subject: [PATCH 2/2] Fix downloadnetfilerawdata import by standardizing schema

The column headers in data/netfile/csv/netfile_cal201_transaction.csv
did not match the table schema, resulting in an import error.

I updated the corresponding model to match.
---
 .../migrations/0002_auto_20160814_2012.py     | 38 +++++++++++++++++++
 netfile_raw/models.py                         | 14 +++++--
 2 files changed, 49 insertions(+), 3 deletions(-)
 create mode 100644 netfile_raw/migrations/0002_auto_20160814_2012.py

diff --git a/netfile_raw/migrations/0002_auto_20160814_2012.py b/netfile_raw/migrations/0002_auto_20160814_2012.py
new file mode 100644
index 00000000..af8f2fc4
--- /dev/null
+++ b/netfile_raw/migrations/0002_auto_20160814_2012.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('netfile_raw', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filerLocalId',
+            field=models.CharField(default='Unknown', max_length=32, db_column='filerLocalId'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filerStateId',
+            field=models.CharField(default='Unknown', max_length=32, db_column='filerStateId'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filingEndDate',
+            field=models.DateField(default='1970-01-01', db_column='filingEndDate'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='netfilecal201transaction',
+            name='filingStartDate',
+            field=models.DateField(default='1970-01-01', db_column='filingStartDate'),
+            preserve_default=False,
+        ),
+    ]
diff --git a/netfile_raw/models.py b/netfile_raw/models.py
index 2eef63c8..1a9f0243 100644
--- a/netfile_raw/models.py
+++ b/netfile_raw/models.py
@@ -36,9 +36,7 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     """
     Netfile 201 data.
     """
-    agency_shortcut = models.CharField(
-        max_length=32,
-        db_column='agency_shortcut')
+    agency_shortcut = models.CharField(max_length=32, db_column='agency_shortcut')
     filerId = models.CharField(
         max_length=32,
         db_column='filerId')
@@ -75,6 +73,8 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     tran_NamT = models.CharField(
         max_length=256,
         db_column='tran_NamT')
+    filingStartDate = models.DateField(
+        db_column='filingStartDate')
     tran_Dscr = models.CharField(
         max_length=256,
         db_column='tran_Dscr')
@@ -150,6 +150,8 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     intr_NamF = models.CharField(
         max_length=256,
         db_column='intr_NamF')
+    filingEndDate = models.DateField(
+        db_column='filingEndDate')
     transactionType = models.IntegerField(
         null=True,
         db_column='transactionType')
@@ -255,6 +257,9 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     int_CmteId = models.CharField(
         max_length=32,
         db_column='int_CmteId')
+    filerLocalId = models.CharField(
+        max_length=32,
+        db_column='filerLocalId')
     calculated_Date = models.DateField(
         null=True,
         db_column='calculated_Date')
@@ -288,6 +293,9 @@ class NetFileCal201Transaction(CalAccessBaseModel):
     entity_Cd = models.CharField(
         max_length=3,
         db_column='entity_Cd')
+    filerStateId = models.CharField(
+        db_column='filerStateId',
+        max_length=32)
     tres_Adr1 = models.CharField(
         max_length=256,
         db_column='tres_Adr1')