From 03c5f7e8c787fcbc4c1e2bc8a8921ce3165e99ec Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 31 Aug 2023 16:06:06 +0100 Subject: [PATCH] refactor(Various update to unit tests): --- .../conveyal/gtfs/loader/JdbcGtfsLoader.java | 8 ++++++- .../java/com/conveyal/gtfs/loader/Table.java | 6 ++--- .../java/com/conveyal/gtfs/GTFSFeedTest.java | 2 +- src/test/java/com/conveyal/gtfs/GTFSTest.java | 23 ++++++++++++++++++- .../fake-agency/datatools_patterns.txt | 2 +- 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java index 3b255f4f9..0804926e9 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java @@ -353,7 +353,13 @@ private int loadInternal(Table table) throws Exception { // SQLite also doesn't support schemas, but you can attach additional database files with schema-like naming. // We'll just literally prepend feed identifiers to table names when supplied. // Some databases require the table to exist before a statement can be prepared. - targetTable.createSqlTable(connection); + if (table.name.equals("patterns")) { + // When creating the patterns table the id field must be flagged as serial and not bigint. This then allows + // the addition of new patterns in PatternBuilder#processPatternAndPatternStops. + targetTable.createSqlTable(connection, true); + } else { + targetTable.createSqlTable(connection); + } // TODO are we loading with or without a header row in our Postgres text file? if (postgresText) { diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index 2e179ae15..cf9b0fe70 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -255,9 +255,9 @@ public Table (String name, Class entityClass, Requirement requ new StringField("name", OPTIONAL), // Editor-specific fields. // direction_id and shape_id are exemplar fields applied to all trips for a pattern. - new ShortField("direction_id", EDITOR, 1), - new ShortField("use_frequency", EDITOR, 1), - new StringField("shape_id", EDITOR).isReferenceTo(SHAPES) + new ShortField("direction_id", OPTIONAL, 1), + new ShortField("use_frequency", OPTIONAL, 1), + new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES) ) .addPrimaryKey() .addPrimaryKeyNames("pattern_id") diff --git a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java index fb8018a4f..e92c54167 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java @@ -134,7 +134,7 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { new DataExpectation[]{ new DataExpectation("pattern_id", "1"), new DataExpectation("route_id", "1"), - new DataExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (3 trips)"), + new DataExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips)"), new DataExpectation("direction_id", "0"), new DataExpectation("shape_id", "5820f377-f947-4728-ac29-ac0102cbc34e") } diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index 19b0413d8..0124c020a 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -109,6 +109,7 @@ public void canLoadAndExportSimpleAgency() { ErrorExpectation[] fakeAgencyErrorExpectations = ErrorExpectation.list( new ErrorExpectation(NewGTFSErrorType.MISSING_FIELD), new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), + new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED), new ErrorExpectation(NewGTFSErrorType.STOP_UNUSED, equalTo("1234567")), @@ -996,6 +997,15 @@ private static int countValidationErrorsOfType( return errorCount; } + /** + * Proprietary table file names are prefix with "datatools_" to distinguish them from GTFS spec files. + */ + private String getTableFileName(String tableName) { + return (tableName.equals("patterns")) + ? String.format("datatools_%s.txt", tableName) + : tableName + ".txt"; + } + /** * Helper to assert that the GTFS that was exported to a zip file matches all data expectations defined in the * persistence expectations. @@ -1014,7 +1024,7 @@ private void assertThatExportedGtfsMeetsExpectations( if (persistenceExpectation.appliesToEditorDatabaseOnly) continue; // No need to check that errors were exported because it is an internal table only. if ("errors".equals(persistenceExpectation.tableName)) continue; - final String tableFileName = persistenceExpectation.tableName + ".txt"; + final String tableFileName = getTableFileName(persistenceExpectation.tableName); LOG.info(String.format("reading table: %s", tableFileName)); ZipEntry entry = gtfsZipfile.getEntry(tableFileName); @@ -1268,6 +1278,17 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("route_color", "7CE6E7") } ), + new PersistenceExpectation( + "patterns", + new RecordExpectation[]{ + new RecordExpectation("pattern_id", "1"), + new RecordExpectation("route_id", "1"), + new RecordExpectation("name", "2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips)"), + new RecordExpectation("direction_id", "0"), + new RecordExpectation("use_frequency", null), + new RecordExpectation("shape_id", "5820f377-f947-4728-ac29-ac0102cbc34e") + } + ), new PersistenceExpectation( "shapes", new RecordExpectation[]{ diff --git a/src/test/resources/fake-agency/datatools_patterns.txt b/src/test/resources/fake-agency/datatools_patterns.txt index 36c7ce5d4..7e72c5ac0 100644 --- a/src/test/resources/fake-agency/datatools_patterns.txt +++ b/src/test/resources/fake-agency/datatools_patterns.txt @@ -1,2 +1,2 @@ pattern_id,route_id,name,direction_id,use_frequency,shape_id -1,1,2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (3 trips),0,,5820f377-f947-4728-ac29-ac0102cbc34e +1,1,2 stops from Butler Ln to Scotts Valley Dr & Victor Sq (1 trips),0,,5820f377-f947-4728-ac29-ac0102cbc34e