Skip to content

Commit

Permalink
refactor(Addressed PR feedback): Update table file name creation
Browse files Browse the repository at this point in the history
  • Loading branch information
br648 committed Sep 5, 2023
1 parent 5ccdc8b commit e648c38
Show file tree
Hide file tree
Showing 8 changed files with 41 additions and 33 deletions.
6 changes: 3 additions & 3 deletions src/main/java/com/conveyal/gtfs/PatternBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ public PatternBuilder(Feed feed) throws SQLException {
}

public void create(Map<TripPatternKey, Pattern> patterns, Set<String> patternIdsLoadedFromFile) {
String patternsTableName = feed.getTableName("patterns");
String tripsTableName = feed.getTableName("trips");
String patternStopsTableName = feed.getTableName("pattern_stops");
String patternsTableName = feed.getTableNameWithSchemaPrefix("patterns");
String tripsTableName = feed.getTableNameWithSchemaPrefix("trips");
String patternStopsTableName = feed.getTableNameWithSchemaPrefix("pattern_stops");

Table patternsTable = new Table(patternsTableName, Pattern.class, Requirement.EDITOR, Table.PATTERNS.fields);
Table patternStopsTable = new Table(patternStopsTableName, PatternStop.class, Requirement.EDITOR, Table.PATTERN_STOP.fields);
Expand Down
36 changes: 18 additions & 18 deletions src/main/java/com/conveyal/gtfs/loader/Feed.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class Feed {

// The unique database schema name for this particular feed, including the separator character (dot).
// This may be the empty string if the feed is stored in the root ("public") schema.
private final String tablePrefix;
private final String databaseSchemaPrefix;

public final TableReader<Agency> agencies;
public final TableReader<Calendar> calendars;
Expand All @@ -44,23 +44,23 @@ public class Feed {
/**
* Create a feed that reads tables over a JDBC connection. The connection should already be set to the right
* schema within the database.
* @param tablePrefix the unique prefix for the table (may be null for no prefix)
* @param databaseSchemaPrefix the unique prefix for the table (may be null for no prefix)
*/
public Feed (DataSource dataSource, String tablePrefix) {
public Feed (DataSource dataSource, String databaseSchemaPrefix) {
this.dataSource = dataSource;
// Ensure separator dot is present
if (tablePrefix != null && !tablePrefix.endsWith(".")) tablePrefix += ".";
this.tablePrefix = tablePrefix == null ? "" : tablePrefix;
agencies = new JDBCTableReader(Table.AGENCY, dataSource, tablePrefix, EntityPopulator.AGENCY);
fareAttributes = new JDBCTableReader(Table.FARE_ATTRIBUTES, dataSource, tablePrefix, EntityPopulator.FARE_ATTRIBUTE);
frequencies = new JDBCTableReader(Table.FREQUENCIES, dataSource, tablePrefix, EntityPopulator.FREQUENCY);
calendars = new JDBCTableReader(Table.CALENDAR, dataSource, tablePrefix, EntityPopulator.CALENDAR);
calendarDates = new JDBCTableReader(Table.CALENDAR_DATES, dataSource, tablePrefix, EntityPopulator.CALENDAR_DATE);
routes = new JDBCTableReader(Table.ROUTES, dataSource, tablePrefix, EntityPopulator.ROUTE);
stops = new JDBCTableReader(Table.STOPS, dataSource, tablePrefix, EntityPopulator.STOP);
trips = new JDBCTableReader(Table.TRIPS, dataSource, tablePrefix, EntityPopulator.TRIP);
stopTimes = new JDBCTableReader(Table.STOP_TIMES, dataSource, tablePrefix, EntityPopulator.STOP_TIME);
patterns = new JDBCTableReader(Table.PATTERNS, dataSource, tablePrefix, EntityPopulator.PATTERN);
if (databaseSchemaPrefix != null && !databaseSchemaPrefix.endsWith(".")) databaseSchemaPrefix += ".";
this.databaseSchemaPrefix = databaseSchemaPrefix == null ? "" : databaseSchemaPrefix;
agencies = new JDBCTableReader(Table.AGENCY, dataSource, databaseSchemaPrefix, EntityPopulator.AGENCY);
fareAttributes = new JDBCTableReader(Table.FARE_ATTRIBUTES, dataSource, databaseSchemaPrefix, EntityPopulator.FARE_ATTRIBUTE);
frequencies = new JDBCTableReader(Table.FREQUENCIES, dataSource, databaseSchemaPrefix, EntityPopulator.FREQUENCY);
calendars = new JDBCTableReader(Table.CALENDAR, dataSource, databaseSchemaPrefix, EntityPopulator.CALENDAR);
calendarDates = new JDBCTableReader(Table.CALENDAR_DATES, dataSource, databaseSchemaPrefix, EntityPopulator.CALENDAR_DATE);
routes = new JDBCTableReader(Table.ROUTES, dataSource, databaseSchemaPrefix, EntityPopulator.ROUTE);
stops = new JDBCTableReader(Table.STOPS, dataSource, databaseSchemaPrefix, EntityPopulator.STOP);
trips = new JDBCTableReader(Table.TRIPS, dataSource, databaseSchemaPrefix, EntityPopulator.TRIP);
stopTimes = new JDBCTableReader(Table.STOP_TIMES, dataSource, databaseSchemaPrefix, EntityPopulator.STOP_TIME);
patterns = new JDBCTableReader(Table.PATTERNS, dataSource, databaseSchemaPrefix, EntityPopulator.PATTERN);
}

/**
Expand All @@ -80,7 +80,7 @@ public ValidationResult validate (FeedValidatorCreator... additionalValidators)
// Reconnect to the existing error tables.
SQLErrorStorage errorStorage;
try {
errorStorage = new SQLErrorStorage(dataSource.getConnection(), tablePrefix, false);
errorStorage = new SQLErrorStorage(dataSource.getConnection(), databaseSchemaPrefix, false);
} catch (SQLException | InvalidNamespaceException ex) {
throw new StorageException(ex);
}
Expand Down Expand Up @@ -157,7 +157,7 @@ public Connection getConnection() throws SQLException {
/**
* @return the table name prefixed with this feed's database schema.
*/
public String getTableName(String tableName) {
return tablePrefix + tableName;
public String getTableNameWithSchemaPrefix(String tableName) {
return databaseSchemaPrefix + tableName;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ private TableLoadResult export (Table table, String filterSql) {
}

// Create entry for table
String textFileName = Table.getTableFileName(table.name);
String textFileName = Table.getTableFileNameWithExtension(table.name);
ZipEntry zipEntry = new ZipEntry(textFileName);
zipOutputStream.putNextEntry(zipEntry);

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ private int getTableSize(Table table) {
private int loadInternal(Table table) throws Exception {
CsvReader csvReader = table.getCsvReader(zip, errorStorage);
if (csvReader == null) {
LOG.info("File {} not found in gtfs zip file.", Table.getTableFileName(table.name));
LOG.info("File {} not found in gtfs zip file.", Table.getTableFileNameWithExtension(table.name));
// This GTFS table could not be opened in the zip, even in a subdirectory.
if (table.isRequired()) errorStorage.storeError(NewGTFSError.forTable(table, MISSING_TABLE));
return 0;
Expand Down
16 changes: 12 additions & 4 deletions src/main/java/com/conveyal/gtfs/loader/Table.java
Original file line number Diff line number Diff line change
Expand Up @@ -622,13 +622,21 @@ public String generateInsertSql (String namespace, boolean setDefaultId) {
);
}

public static String getTableFileNameWithExtension(String tableName) {
return getTableFileName(tableName, ".txt");
}

public static String getTableFileName(String tableName) {
return getTableFileName(tableName, "");
}

/**
* Proprietary table file names are prefix with "datatools_" to distinguish them from GTFS spec files.
*/
public static String getTableFileName(String tableName) {
private static String getTableFileName(String tableName, String fileExtension) {
return (tableName.equals("patterns"))
? String.format("%s%s.txt", PROPRIETARY_FILE_PREFIX, tableName)
: tableName + ".txt";
? String.format("%s%s%s", PROPRIETARY_FILE_PREFIX, tableName, fileExtension)
: String.format("%s%s", tableName, fileExtension);
}

/**
Expand All @@ -638,7 +646,7 @@ public static String getTableFileName(String tableName) {
* It then creates a CSV reader for that table if it's found.
*/
public CsvReader getCsvReader(ZipFile zipFile, SQLErrorStorage sqlErrorStorage) {
final String tableFileName = getTableFileName(this.name);
final String tableFileName = getTableFileNameWithExtension(this.name);
ZipEntry entry = zipFile.getEntry(tableFileName);
if (entry == null) {
// Table was not found, check if it is in a subdirectory.
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/conveyal/gtfs/model/Pattern.java
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public Pattern () {}
public static class Loader extends Entity.Loader<Pattern> {

public Loader(GTFSFeed feed) {
super(feed, Table.PROPRIETARY_FILE_PREFIX + "patterns");
super(feed, Table.getTableFileName("patterns"));
}

@Override
Expand Down Expand Up @@ -141,7 +141,7 @@ public void loadOneRow() throws IOException {

public static class Writer extends Entity.Writer<Pattern> {
public Writer (GTFSFeed feed) {
super(feed, Table.PROPRIETARY_FILE_PREFIX + "patterns");
super(feed, Table.getTableFileName("patterns"));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ select durations.service_id, duration_seconds, days_active from (
// Except that some service IDs may have no trips on them, or may not be referenced in any calendar or
// calendar exception, which would keep them from appearing in either of those tables. So we just create
// this somewhat redundant materialized view to serve as a master list of all services.
String servicesTableName = feed.getTableName("services");
String servicesTableName = feed.getTableNameWithSchemaPrefix("services");
String sql = String.format("create table %s (service_id varchar, n_days_active integer, duration_seconds integer, n_trips integer)", servicesTableName);
LOG.info(sql);
statement.execute(sql);
Expand All @@ -285,7 +285,7 @@ select durations.service_id, duration_seconds, days_active from (
serviceTracker.executeRemaining();

// Create a table that shows on which dates each service is active.
String serviceDatesTableName = feed.getTableName("service_dates");
String serviceDatesTableName = feed.getTableNameWithSchemaPrefix("service_dates");
sql = String.format("create table %s (service_date varchar, service_id varchar)", serviceDatesTableName);
LOG.info(sql);
statement.execute(sql);
Expand Down Expand Up @@ -318,7 +318,7 @@ select durations.service_id, duration_seconds, days_active from (
// where dates.service_id = durations.service_id
// group by service_date, route_type order by service_date, route_type;

String serviceDurationsTableName = feed.getTableName("service_durations");
String serviceDurationsTableName = feed.getTableNameWithSchemaPrefix("service_durations");
sql = String.format("create table %s (service_id varchar, route_type integer, " +
"duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName);
LOG.info(sql);
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/conveyal/gtfs/GTFSTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -1016,7 +1016,7 @@ private void assertThatExportedGtfsMeetsExpectations(
if (persistenceExpectation.appliesToEditorDatabaseOnly) continue;
// No need to check that errors were exported because it is an internal table only.
if ("errors".equals(persistenceExpectation.tableName)) continue;
final String tableFileName = Table.getTableFileName(persistenceExpectation.tableName);
final String tableFileName = Table.getTableFileNameWithExtension(persistenceExpectation.tableName);
LOG.info(String.format("reading table: %s", tableFileName));

ZipEntry entry = gtfsZipfile.getEntry(tableFileName);
Expand Down

0 comments on commit e648c38

Please sign in to comment.