-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #99 from linkedconnections/development
v1.0.0
- Loading branch information
Showing
18 changed files
with
997 additions
and
419 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,82 @@ | ||
| #!/usr/bin/env node | ||
|
|
||
| var csv = require('fast-csv'), | ||
| fs = require('fs'), | ||
| St2C = require('../lib/stoptimes/st2c.js'); | ||
| const csv = require('fast-csv'); | ||
| const fs = require('fs'); | ||
| const readline = require('readline'); | ||
| const St2C = require('../lib/stoptimes/st2c.js'); | ||
| const numCPUs = require('os').cpus().length; | ||
|
|
||
| var stopTimes = fs.createReadStream('stop_times.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { | ||
| console.error(e); | ||
| // Fragment trips.txt accordingly to the number fo available CPU processors | ||
| var trips = fs.createReadStream('trips.txt', { encoding: 'utf8' }) | ||
| .pipe(csv.parse({ objectMode: true, headers: true, quote: '"' })) | ||
| .on('error', function (e) { | ||
| console.error(e); | ||
| }); | ||
| var tripsPool = createWriteStreams('trips'); | ||
| var tripIndex = -1; | ||
|
|
||
| trips.on('data', trip => { | ||
| if (tripIndex === -1) { | ||
| for (let i in tripsPool) { | ||
| tripsPool[i].write(Object.keys(trip)); | ||
| } | ||
| tripIndex++; | ||
| } | ||
|
|
||
| tripsPool[tripIndex].write(Object.values(trip)); | ||
| tripIndex = tripIndex < numCPUs - 1 ? tripIndex + 1 : 0; | ||
| }); | ||
|
|
||
| trips.on('end', () => { | ||
| for (let i in tripsPool) { | ||
| tripsPool[i].end(); | ||
| } | ||
| }); | ||
|
|
||
| // Also fragment stop_times.txt | ||
| var stopTimes = fs.createReadStream('stop_times.txt', { encoding: 'utf8', objectMode: true }) | ||
| .pipe(csv.parse({ objectMode: true, headers: true, quote: '"' })) | ||
| .on('error', function (e) { | ||
| console.error(e); | ||
| }); | ||
|
|
||
| var connectionsPool = createWriteStreams('connections'); | ||
| var connIndex = -1; | ||
| var currentTrip = null; | ||
| var printedRows = 0; | ||
|
|
||
| var connectionRules = stopTimes.pipe(new St2C()); | ||
|
|
||
| connectionRules.on('data', row => { | ||
| printConnectionRule(row); | ||
| if (connIndex === -1) { | ||
| for (let i in connectionsPool) { | ||
| connectionsPool[i].write(Object.keys(row)); | ||
| } | ||
| } | ||
|
|
||
| if (row['trip_id'] !== currentTrip) { | ||
| currentTrip = row['trip_id']; | ||
| connIndex = connIndex < numCPUs - 1 ? connIndex + 1 : 0; | ||
| } | ||
|
|
||
| connectionsPool[connIndex].write(Object.values(row)) | ||
| printedRows++; | ||
| }); | ||
|
|
||
| connectionRules.on('end', () => { | ||
| console.error('Converted ' + printedRows + ' stoptimes to connections'); | ||
| for (let i in connectionsPool) { | ||
| connectionsPool[i].end(); | ||
| } | ||
| console.error('Converted ' + printedRows + ' stop_times to connections'); | ||
| }); | ||
|
|
||
| var printConnectionHeader = function (row) { | ||
| console.log(Object.keys(row).join(',')); | ||
| }; | ||
|
|
||
| var printedRows = 0; | ||
| var printConnectionRule = function (row) { | ||
| if (printedRows === 0) { | ||
| printConnectionHeader(row); | ||
| function createWriteStreams(name) { | ||
| let writers = []; | ||
| for (let i = 0; i < numCPUs; i++) { | ||
| const stream = csv.format(); | ||
| stream.pipe(fs.createWriteStream(name + '_' + i + '.txt', { encoding: 'utf8' })); | ||
| writers.push(stream); | ||
| } | ||
| console.log(Object.values(row).join(',')); | ||
| printedRows++; | ||
| } | ||
|
|
||
| return writers; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| const { Transform } = require('stream'); | ||
|
|
||
| class Connections2CSV extends Transform { | ||
| constructor(header) { | ||
| super({ objectMode: true }); | ||
| this._headerStreamed = false; | ||
| if(!header) { | ||
| this.headerStreamed = true; | ||
| } | ||
| } | ||
|
|
||
| _transform(connection, encoding, done) { | ||
| if (!this.headerStreamed) { | ||
| this.headerStreamed = true; | ||
| done(null, '"departureStop","departureTime","arrivalStop","arrivalTime","trip","route","headsign"\n'); | ||
| } else { | ||
| let csv = connection["departureStop"] + ',' + connection["departureTime"].toISOString() + ',' | ||
| + connection["arrivalStop"] + ',' + connection["arrivalTime"].toISOString() + ',' + connection["trip"]["trip_id"] + ',' | ||
| + connection.trip.route.route_id + ',"' + connection.headsign + '"' + '\n'; | ||
| done(null, csv); | ||
| } | ||
| } | ||
|
|
||
| get headerStreamed() { | ||
| return this._headerStreamed; | ||
| } | ||
|
|
||
| set headerStreamed(value) { | ||
| this._headerStreamed = value; | ||
| } | ||
| } | ||
|
|
||
| module.exports = Connections2CSV; |
Oops, something went wrong.