refactor(Various update to unit tests):

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java

@@ -353,7 +353,13 @@ private int loadInternal(Table table) throws Exception {
         // SQLite also doesn't support schemas, but you can attach additional database files with schema-like naming.
         // We'll just literally prepend feed identifiers to table names when supplied.
         // Some databases require the table to exist before a statement can be prepared.
-        targetTable.createSqlTable(connection);
+        if (table.name.equals("patterns")) {
+            // When creating the patterns table the id field must be flagged as serial and not bigint. This then allows
+            // the addition of new patterns in PatternBuilder#processPatternAndPatternStops.
+            targetTable.createSqlTable(connection, true);
+        } else {
+            targetTable.createSqlTable(connection);
+        }
 
         // TODO are we loading with or without a header row in our Postgres text file?
         if (postgresText) {

src/main/java/com/conveyal/gtfs/loader/Table.java

@@ -255,9 +255,9 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
             new StringField("name", OPTIONAL),
             // Editor-specific fields.
             // direction_id and shape_id are exemplar fields applied to all trips for a pattern.
-            new ShortField("direction_id", EDITOR, 1),
-            new ShortField("use_frequency", EDITOR, 1),
-            new StringField("shape_id", EDITOR).isReferenceTo(SHAPES)
+            new ShortField("direction_id", OPTIONAL, 1),
+            new ShortField("use_frequency", OPTIONAL, 1),
+            new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES)
     )
     .addPrimaryKey()
     .addPrimaryKeyNames("pattern_id")

src/test/java/com/conveyal/gtfs/GTFSFeedTest.java

@@ -134,7 +134,7 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException {
                 new DataExpectation[]{
                     new DataExpectation("pattern_id", "1"),
                     new DataExpectation("route_id", "1"),
-                    new DataExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (3 trips)"),
+                    new DataExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips)"),
                     new DataExpectation("direction_id", "0"),
                     new DataExpectation("shape_id", "5820f377-f947-4728-ac29-ac0102cbc34e")
                 }

src/test/java/com/conveyal/gtfs/GTFSTest.java

@@ -109,6 +109,7 @@ public void canLoadAndExportSimpleAgency() {
         ErrorExpectation[] fakeAgencyErrorExpectations = ErrorExpectation.list(
             new ErrorExpectation(NewGTFSErrorType.MISSING_FIELD),
             new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY),
+            new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY),
             new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME),
             new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED),
             new ErrorExpectation(NewGTFSErrorType.STOP_UNUSED, equalTo("1234567")),
@@ -996,6 +997,15 @@ private static int countValidationErrorsOfType(
         return errorCount;
     }
 
+    /**
+     * Proprietary table file names are prefix with "datatools_" to distinguish them from GTFS spec files.
+     */
+    private String getTableFileName(String tableName) {
+        return (tableName.equals("patterns"))
+            ? String.format("datatools_%s.txt", tableName)
+            : tableName + ".txt";
+    }
+
     /**
      * Helper to assert that the GTFS that was exported to a zip file matches all data expectations defined in the
      * persistence expectations.
@@ -1014,7 +1024,7 @@ private void assertThatExportedGtfsMeetsExpectations(
             if (persistenceExpectation.appliesToEditorDatabaseOnly) continue;
             // No need to check that errors were exported because it is an internal table only.
             if ("errors".equals(persistenceExpectation.tableName)) continue;
-            final String tableFileName = persistenceExpectation.tableName + ".txt";
+            final String tableFileName = getTableFileName(persistenceExpectation.tableName);
             LOG.info(String.format("reading table: %s", tableFileName));
 
             ZipEntry entry = gtfsZipfile.getEntry(tableFileName);
@@ -1268,6 +1278,17 @@ private void assertThatPersistenceExpectationRecordWasFound(
                 new RecordExpectation("route_color", "7CE6E7")
             }
         ),
+        new PersistenceExpectation(
+            "patterns",
+            new RecordExpectation[]{
+                new RecordExpectation("pattern_id", "1"),
+                new RecordExpectation("route_id", "1"),
+                new RecordExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips)"),
+                new RecordExpectation("direction_id", "0"),
+                new RecordExpectation("use_frequency", null),
+                new RecordExpectation("shape_id", "5820f377-f947-4728-ac29-ac0102cbc34e")
+            }
+        ),
         new PersistenceExpectation(
             "shapes",
             new RecordExpectation[]{

src/test/resources/fake-agency/datatools_patterns.txt

@@ -1,2 +1,2 @@
 pattern_id,route_id,name,direction_id,use_frequency,shape_id
-1,1,2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (3 trips),0,,5820f377-f947-4728-ac29-ac0102cbc34e
+1,1,2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips),0,,5820f377-f947-4728-ac29-ac0102cbc34e