Skip to content

Commit

Permalink
refactor(To allow for unique schedule exception names): Combine sched…
Browse files Browse the repository at this point in the history 
…ule exceptions for a single ser
  • Loading branch information
Robin Beer authored and Robin Beer committed Aug 29, 2023
1 parent b00499a commit 1b6cc2f
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 38 deletions.
44 changes: 22 additions & 22 deletions src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.LocalDate;
Expand Down Expand Up @@ -93,7 +94,7 @@ public FeedLoadResult exportTables() {
FeedLoadResult result = new FeedLoadResult();

try {
zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile));
zipOutputStream = new ZipOutputStream(Files.newOutputStream(Paths.get(outFile)));
long startTime = System.currentTimeMillis();
// We get a single connection object and share it across several different methods.
// This ensures that actions taken in one method are visible to all subsequent SQL statements.
Expand Down Expand Up @@ -126,16 +127,14 @@ public FeedLoadResult exportTables() {
if (fromEditor) {
// Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot.
GTFSFeed feed = new GTFSFeed();
// FIXME: The below table readers should probably just share a connection with the exporter.
JDBCTableReader<ScheduleException> exceptionsReader =
new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".",
EntityPopulator.SCHEDULE_EXCEPTION);
JDBCTableReader<ScheduleException> exceptionsReader =new JDBCTableReader(
Table.SCHEDULE_EXCEPTIONS,
dataSource,
feedIdToExport + ".",
EntityPopulator.SCHEDULE_EXCEPTION
);
JDBCTableReader<Calendar> calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToExport);
Iterable<Calendar> calendars = calendarReader.getAll();
Set<String> calendarServiceIds = new HashSet<>();
for (Calendar calendar : calendarReader.getAll()) {
calendarServiceIds.add(calendar.service_id);
}
Iterable<ScheduleException> exceptionsIterator = exceptionsReader.getAll();
List<ScheduleException> calendarExceptions = new ArrayList<>();
List<ScheduleException> calendarDateExceptions = new ArrayList<>();
Expand All @@ -149,20 +148,21 @@ public FeedLoadResult exportTables() {
}

int calendarDateCount = calendarDateExceptions.size();
// Extract calendar date services, convert to calendar date and add to the feed. We are expecting only one
// date which will be exported as a service on the specified date.
// Extract calendar date services, convert to calendar date and add to the feed.
for (ScheduleException ex : calendarDateExceptions) {
// Only ever expecting one date here.
LocalDate date = ex.dates.get(0);
CalendarDate calendarDate = new CalendarDate();
calendarDate.date = date;
calendarDate.service_id = ex.name;
calendarDate.exception_type = 1;
Service service = new Service(calendarDate.service_id);
service.calendar_dates.put(date, calendarDate);
// If the calendar dates provided contain duplicates (e.g. two or more identical service ids that are
// NOT associated with a calendar) only the first entry will persist export.
feed.services.put(calendarDate.service_id, service);
for (LocalDate date : ex.dates) {
String serviceId = ex.customSchedule.get(0);
CalendarDate calendarDate = new CalendarDate();
calendarDate.date = date;
calendarDate.service_id = serviceId;
calendarDate.exception_type = 1;
Service service = new Service(serviceId);
service.calendar_dates.put(date, calendarDate);
// If the calendar dates provided contain duplicates (e.g. two or more identical service ids
// that are NOT associated with a calendar) only the first entry would persist export. To
// resolve this a unique key consisting of service id and date is used.
feed.services.put(String.format("%s-%s", calendarDate.service_id, calendarDate.date), service);
}
}

// check whether the feed is organized in a format with the calendars.txt file
Expand Down
34 changes: 26 additions & 8 deletions src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ private TableLoadResult createScheduleExceptionsTable() {
// Iterate through calendar dates to build up appropriate service dates.
Multimap<String, String> removedServiceForDate = HashMultimap.create();
Multimap<String, String> addedServiceForDate = HashMultimap.create();
HashMap<String, String> calendarDateService = new HashMap<>();
HashMap<String, Set<String>> calendarDateService = new HashMap<>();
for (CalendarDate calendarDate : calendarDates) {
// Skip any null dates or service ids.
if (calendarDate.date == null || calendarDate.service_id == null) {
Expand All @@ -251,13 +251,20 @@ private TableLoadResult createScheduleExceptionsTable() {
removedServiceForDate.put(date, calendarDate.service_id);
}
} else {
// Calendar date is unique.
calendarDateService.put(date, calendarDate.service_id);
// Calendar date is not related to a calendar. Group calendar dates by service id.
if (calendarDateService.containsKey(calendarDate.service_id)) {
calendarDateService.get(calendarDate.service_id).add(date);
} else {
Set<String> dates = new HashSet<>();
dates.add(date);
calendarDateService.put(calendarDate.service_id, dates);
}

}
}

String sql = String.format(
"insert into %s (name, dates, exemplar, added_service, removed_service) values (?, ?, ?, ?, ?)",
"insert into %s (name, dates, exemplar, custom_schedule, added_service, removed_service) values (?, ?, ?, ?, ?, ?)",
scheduleExceptionsTableName
);
PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql);
Expand All @@ -276,17 +283,23 @@ private TableLoadResult createScheduleExceptionsTable() {
date,
new String[] {date},
ScheduleException.ExemplarServiceDescriptor.SWAP,
new String[] {},
addedServiceForDate.get(date).toArray(),
removedServiceForDate.get(date).toArray()
);
}
for (Map.Entry<String,String> entry : calendarDateService.entrySet()) {

for (Map.Entry<String,Set<String>> entry : calendarDateService.entrySet()) {
String serviceId = entry.getKey();
String[] dates = entry.getValue().toArray(new String[0]);
createScheduledExceptionStatement(
scheduleExceptionsStatement,
scheduleExceptionsTracker,
entry.getValue(),
new String[] {entry.getKey()},
// Unique-ish schedule name that shouldn't conflict with existing service ids.
String.format("%s-%s", serviceId, dates[0]),
dates,
ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE,
new String[] {serviceId},
new String[] {},
new String[] {}
);
Expand Down Expand Up @@ -369,6 +382,7 @@ private void createScheduledExceptionStatement(
String name,
String[] dates,
ScheduleException.ExemplarServiceDescriptor exemplarServiceDescriptor,
Object[] customSchedule,
Object[] addedServicesForDate,
Object[] removedServicesForDate
) throws SQLException {
Expand All @@ -377,10 +391,14 @@ private void createScheduledExceptionStatement(
scheduleExceptionsStatement.setInt(3, exemplarServiceDescriptor.getValue());
scheduleExceptionsStatement.setArray(
4,
connection.createArrayOf("text", addedServicesForDate)
connection.createArrayOf("text", customSchedule)
);
scheduleExceptionsStatement.setArray(
5,
connection.createArrayOf("text", addedServicesForDate)
);
scheduleExceptionsStatement.setArray(
6,
connection.createArrayOf("text", removedServicesForDate)
);
scheduleExceptionsTracker.addBatch();
Expand Down
7 changes: 1 addition & 6 deletions src/main/java/com/conveyal/gtfs/model/Calendar.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,7 @@ protected void writeOneRow(Calendar c) throws IOException {
@Override
protected Iterator<Calendar> iterator() {
// wrap an iterator over services
Iterator<Calendar> calIt = Iterators.transform(feed.services.values().iterator(), new Function<Service, Calendar> () {
@Override
public Calendar apply (Service s) {
return s.calendar;
}
});
Iterator<Calendar> calIt = Iterators.transform(feed.services.values().iterator(), s -> s.calendar);

// not every service has a calendar (e.g. TriMet has no calendars, just calendar dates).
// This is legal GTFS, so skip services with no calendar
Expand Down
23 changes: 22 additions & 1 deletion src/test/java/com/conveyal/gtfs/GTFSTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,26 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() {
new RecordExpectation("exception_type", 1)
}
),
new PersistenceExpectation(
"calendar_dates",
new RecordExpectation[]{
new RecordExpectation(
"service_id", "calendar-dates-txt-service-three"
),
new RecordExpectation("date", 20170917),
new RecordExpectation("exception_type", 1)
}
),
new PersistenceExpectation(
"calendar_dates",
new RecordExpectation[]{
new RecordExpectation(
"service_id", "calendar-dates-txt-service-three"
),
new RecordExpectation("date", 20170918),
new RecordExpectation("exception_type", 1)
}
),
new PersistenceExpectation(
"stop_times",
new RecordExpectation[]{
Expand Down Expand Up @@ -528,7 +548,8 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() {
new ErrorExpectation(NewGTFSErrorType.MISSING_FIELD),
new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME),
new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME),
new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED)
new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED),
new ErrorExpectation(NewGTFSErrorType.SERVICE_UNUSED)
);
assertThat(
runIntegrationTestOnFolder(
Expand Down
4 changes: 3 additions & 1 deletion src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@ service_id,date,exception_type
in-both-calendar-txt-and-calendar-dates,20170920,2
only-in-calendar-dates-txt,20170916,1
calendar-dates-txt-service-one,20170917,1
calendar-dates-txt-service-two,20170918,1
calendar-dates-txt-service-two,20170918,1
calendar-dates-txt-service-three,20170917,1
calendar-dates-txt-service-three,20170918,1

0 comments on commit 1b6cc2f

Please sign in to comment.