From 74d03644ad4607b02c468ced6e19e0ae3b3b9b5b Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Fri, 14 Feb 2020 15:35:31 +0100 Subject: [PATCH 1/6] Rely on Node.js Worker Threads --- bin/gtfs2lc-sort.sh | 17 +- bin/gtfs2lc.js | 126 +++----- bin/linkedconnections-sort.sh | 13 +- bin/stoptimes2connections.js | 87 ++++-- lib/Connections2CSV.js | 33 ++ lib/Connections2JSONLD.js | 86 +++--- lib/Connections2Triples.js | 6 +- lib/ConnectionsBuilder.js | 5 +- lib/URIStrategy.js | 11 +- lib/gtfs2connections.js | 297 +++++++++++++++--- lib/stores/LevelStore.js | 36 --- lib/stores/MemStore.js | 37 --- lib/stores/Store.js | 15 +- package-lock.json | 554 +++++++++++++++++++--------------- package.json | 6 +- test/resultStream.test.js | 80 ++--- 16 files changed, 819 insertions(+), 590 deletions(-) create mode 100644 lib/Connections2CSV.js delete mode 100644 lib/stores/LevelStore.js delete mode 100644 lib/stores/MemStore.js diff --git a/bin/gtfs2lc-sort.sh b/bin/gtfs2lc-sort.sh index 5a9ccd5..4aed1ff 100755 --- a/bin/gtfs2lc-sort.sh +++ b/bin/gtfs2lc-sort.sh @@ -27,19 +27,16 @@ CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" TRIPID_TRIPS=`head -n1 trips.txt | tr "," "\n" | grep -n "trip_id"| cut -d: -f1` TRIPID_STOPTIMES=`head -n1 stop_times.txt | tr "," "\n" | grep -n "trip_id"| cut -d: -f1` STOPSEQUENCE_STOPTIMES=`head -n1 stop_times.txt | tr "," "\n" | grep -n "stop_sequence"| cut -d: -f1` - ## sort stop_times.txt by trip id and stop sequence + ## Sort stop_times.txt by trip id and stop sequence { head -n 1 stop_times.txt ; tail -n +2 stop_times.txt | sort -t , -k ${TRIPID_STOPTIMES}d,${TRIPID_STOPTIMES} -k${STOPSEQUENCE_STOPTIMES}n,${STOPSEQUENCE_STOPTIMES}; } > stop_times2.txt ; mv stop_times2.txt stop_times.txt ; - ## use stoptimes2connections to create a connections CSV file instead - echo Creating connections.txt file - $CURDIR/stoptimes2connections.js > connections.txt; - - ## order the connections.txt according to deptime, artime and stops - TRIPID_CONNECTIONS=`head -n1 connections.txt | tr "," "\n" | grep -n "trip_id"| cut -d: -f1` + ## Sort trips.txt by trip_id and have the same ordering as stop_times.txt + { head -n 1 trips.txt ; tail -n +2 trips.txt | sort -t , -k ${TRIPID_TRIPS}d,${TRIPID_TRIPS} ; } > trips2.txt ; mv trips2.txt trips.txt & + ## Use stoptimes2connections to create a set of connections and trips files + echo Creating connections and trips files according to the number of CPU processors available + $CURDIR/stoptimes2connections.js; - ## Finally sort all files in order to be processed for gtfs2lc + ## Finally sort calendar.txt and calendar_dates.txt files in order to be processed for gtfs2lc echo Sorting files in directory $1; - { head -n 1 connections.txt ; tail -n +2 connections.txt | sort -t , -k ${TRIPID_CONNECTIONS}d,${TRIPID_CONNECTIONS} ; } > connections2.txt ; mv connections2.txt connections.txt & - { head -n 1 trips.txt ; tail -n +2 trips.txt | sort -t , -k ${TRIPID_TRIPS}d,${TRIPID_TRIPS} ; } > trips2.txt ; mv trips2.txt trips.txt & { head -n 1 calendar.txt ; tail -n +2 calendar.txt | sort -t , -k 1d,1; } > calendar2.txt ; mv calendar2.txt calendar.txt & { head -n 1 calendar_dates.txt ; tail -n +2 calendar_dates.txt | sort -t , -k 1d,1; } > calendar_dates2.txt ; mv calendar_dates2.txt calendar_dates.txt & } ; diff --git a/bin/gtfs2lc.js b/bin/gtfs2lc.js index 35dbe31..4a91da4 100755 --- a/bin/gtfs2lc.js +++ b/bin/gtfs2lc.js @@ -1,34 +1,18 @@ #!/usr/bin/env node var program = require('commander'), - gtfs2lc = require('../lib/gtfs2lc.js'), - MongoStream = require('../lib/Connections2Mongo.js'), - N3 = require('n3'), - jsonldstream = require('jsonld-stream'), - Connections2JSONLD = require('../lib/Connections2JSONLD.js'), - fs = require('fs'); - -//ty http://www.geedew.com/remove-a-directory-that-is-not-empty-in-nodejs/ -var deleteFolderRecursive = function(path) { - if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file,index){ - var curPath = path + "/" + file; - if(fs.lstatSync(curPath).isDirectory()) { // recurse - deleteFolderRecursive(curPath); - } else { // delete file - fs.unlinkSync(curPath); - } - }); - fs.rmdirSync(path); - } -}; + gtfs2lc = require('../lib/gtfs2lc.js'), + fs = require('fs'), + del = require('del'); console.error("GTFS to linked connections converter use --help to discover more functions"); program - .option('-f, --format ', 'Format of the output. Possibilities: csv, ntriples, turtle, json, jsonld (default: json), mongo (extended JSON format to be used with mongoimport) or mongold') - .option('-b, --baseUris ', 'path to a file that describes the baseUris in json') - .option('-S, --store ', 'store type: LevelStore (uses your harddisk - for if you run out of RAM) or MemStore (default)') + .option('-f, --format ', 'Format of the output. Possibilities: csv, n-triples, turtle, json, jsonld, mongo (extended JSON format to be used with mongoimport) or mongold (default: json)') + .option('-b, --baseUris ', 'Path to a file that describes the baseUris in json') + .option('-o, --output ', 'Path to the location where the result file will be stored') + .option('-s, --stream', 'Get the connections as a stream on the standard output') + .option('-S, --store ', 'Store type: KeyvStore (uses your disk to avoid that you run out of RAM) or MemStore (default)') .arguments('', 'Path to sorted GTFS files') .action(function (path) { program.path = path; @@ -40,77 +24,33 @@ if (!program.path) { process.exit(); } -var mapper = new gtfs2lc.Connections({ - store : program.store -}); +if (program.path.endsWith('/')) { + program.path = program.path.slice(0, -1); +} + +var output = program.output || program.path; +if (output.endsWith('/')) { + output = output.slice(0, -1); +} var baseUris = null; if (program.baseUris) { baseUris = JSON.parse(fs.readFileSync(program.baseUris, 'utf-8')); } +var mapper = new gtfs2lc.Connections({ + store: !program.store || program.store === 'undefined' ? 'MemStore' : program.store, + format: !program.format || program.format === 'undefined' ? 'json' : program.format, + baseUris: baseUris +}); + var resultStream = null; -mapper.resultStream(program.path, function (stream, stopsdb) { - resultStream = stream; - if (!program.format || program.format === "json") { - stream.on('data', function (connection) { - console.log(JSON.stringify(connection)); - }) - } else if (program.format === 'mongo') { - stream.pipe(new MongoStream()).on('data', function (connection) { - console.log(JSON.stringify(connection)); - }); - } else if (program.format === 'csv') { - //print header - console.error('The CSV output is not using a Linked Data format – jsonld is the preferred format.'); - console.log('"id","departureStop","departureTime","arrivalStop","arrivalTime","trip","route","headsign"'); - var count = 0; - - stream.on('data', function (connection) { - console.log(count + ',' + connection["departureStop"] + ',' + connection["departureTime"].toISOString() + ',' + connection["arrivalStop"] + ',' + connection["arrivalTime"].toISOString() + ',' + connection["trip"]["trip_id"] + ',' + connection.trip.route.route_id + ',"' + connection.headsign + '"'); - count ++; - }); - } else if (['jsonld','mongold'].indexOf(program.format) > -1) { - var context = { - '@context' : { - lc: 'http://semweb.mmlab.be/ns/linkedconnections#', - gtfs : 'http://vocab.gtfs.org/terms#', - xsd: 'http://www.w3.org/2001/XMLSchema#', - trip : { '@type' : '@id', '@id' : 'gtfs:trip' }, - Connection : 'lc:Connection', - CancelledConnection: 'lc:CancelledConnection', - departureTime : { '@type' : 'xsd:dateTime', '@id' : 'lc:departureTime' }, - departureStop : { '@type' : '@id', '@id' : 'lc:departureStop' }, - arrivalStop : { '@type' : '@id', '@id' : 'lc:arrivalStop' }, - arrivalTime : { '@type' : 'xsd:dateTime', '@id' : 'lc:arrivalTime' }, - } - }; - //convert triples stream to jsonld stream - stream = stream.pipe(new Connections2JSONLD(baseUris, stopsdb, context)); - //prepare the output - if (program.format === 'mongold') { - //convert JSONLD Stream to MongoDB Stream - stream = stream.pipe(new MongoStream()); - } - stream = stream.pipe(new jsonldstream.Serializer()).pipe(process.stdout); - } else if (['ntriples','turtle'].indexOf(program.format) > -1) { - stream = stream.pipe(new gtfs2lc.Connections2Triples(baseUris, stopsdb)); - if (program.format === 'ntriples') { - stream = stream.pipe(new N3.StreamWriter({ format : 'N-Triples'})); - } else if (program.format === 'turtle') { - stream = stream.pipe(new N3.StreamWriter({ prefixes: { lc: 'http://semweb.mmlab.be/ns/linkedconnections#', gtfs : 'http://vocab.gtfs.org/terms#', xsd: 'http://www.w3.org/2001/XMLSchema#' } })); - } - stream.pipe(process.stdout); +mapper.resultStream(program.path, output, function (path) { + if (program.stream) { + fs.createReadStream(path).pipe(process.stdout); + } else { + console.error('Linked Connections successfully created at ' + path + '!'); } - stream.on('error', error => { - console.error(error); - }); - stream.on('finish', function () { - console.error('Stream ended - everything should be fully converted!'); - //clean up the leveldb - deleteFolderRecursive(program.path + "/.services"); - deleteFolderRecursive(program.path + "/.trips"); - }); }); process.on('SIGINT', function () { @@ -118,8 +58,14 @@ process.on('SIGINT', function () { if (resultStream) { resultStream.end(); } else { - deleteFolderRecursive(program.path + "/.services"); - deleteFolderRecursive(program.path + "/.trips"); + del([ + output + '/.stops', + output + '/.routes', + output + '/.services', + output + '/raw_*' + ], + { force: true }).then(function () { + process.exit(0); + }); } - process.exit(0); }); diff --git a/bin/linkedconnections-sort.sh b/bin/linkedconnections-sort.sh index 5ee8f0c..0f341e8 100755 --- a/bin/linkedconnections-sort.sh +++ b/bin/linkedconnections-sort.sh @@ -8,12 +8,13 @@ CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ## should be 1 argument: the connections.nldjsonld file, and this file should exist [[ $# == 1 ]] && [[ -f $1 ]] && { - ## First order it by departureTime, as well as - DEPARTURETIME=$(( `head -n1 $1 | tr "," "\n" | grep -n "departureTime"| cut -d: -f1` )); - DEPARTURESTOP=$(( `head -n1 $1 | tr "," "\n" | grep -n "departureStop"| cut -d: -f1` )); - ARRIVALTIME=$(( `head -n1 $1 | tr "," "\n" | grep -n "arrivalTime"| cut -d: -f1` )); - ARRIVALSTOP=$(( `head -n1 $1 | tr "," "\n" | grep -n "arrivalStop"| cut -d: -f1` )); - ROUTE=$(( `head -n1 $1 | tr "," "\n" | grep -n "gtfs:route"| cut -d: -f1` )); + ## Skip first line that corresponds to JSON-LD @context + ## Order it by departureTime, as well as + DEPARTURETIME=$(( `sed 1,1d $1 | head -n1 | tr "," "\n" | grep -n "departureTime"| cut -d: -f1` )); + DEPARTURESTOP=$(( `sed 1,1d $1 | head -n1 | tr "," "\n" | grep -n "departureStop"| cut -d: -f1` )); + ARRIVALTIME=$(( `sed 1,1d $1 | head -n1 | tr "," "\n" | grep -n "arrivalTime"| cut -d: -f1` )); + ARRIVALSTOP=$(( `sed 1,1d $1 | head -n1 | tr "," "\n" | grep -n "arrivalStop"| cut -d: -f1` )); + ROUTE=$(( `sed 1,1d $1 | head -n1 | tr "," "\n" | grep -n "gtfs:route"| cut -d: -f1` )); ## And after the sorting, we need to pipe it to a process that is able to join trains. Ordered in descending order, but afterwards again sorted in ascending order sort $1 -t , -k ${DEPARTURETIME}dr,${DEPARTURETIME} -k ${ARRIVALTIME}dr,${ARRIVALTIME} -k ${ROUTE}dr,${ROUTE} -k ${DEPARTURESTOP}dr,${DEPARTURESTOP} -k ${ARRIVALSTOP}dr,${ARRIVALSTOP} | $CURDIR/linkedconnections-sortandjoin.js | sort -t , -k ${DEPARTURETIME}d,${DEPARTURETIME}; diff --git a/bin/stoptimes2connections.js b/bin/stoptimes2connections.js index ec9d66b..fd83799 100755 --- a/bin/stoptimes2connections.js +++ b/bin/stoptimes2connections.js @@ -1,31 +1,82 @@ #!/usr/bin/env node -var csv = require('fast-csv'), - fs = require('fs'), - St2C = require('../lib/stoptimes/st2c.js'); +const csv = require('fast-csv'); +const fs = require('fs'); +const readline = require('readline'); +const St2C = require('../lib/stoptimes/st2c.js'); +const numCPUs = require('os').cpus().length; -var stopTimes = fs.createReadStream('stop_times.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error(e); +// Fragment trips.txt accordingly to the number fo available CPU processors +var trips = fs.createReadStream('trips.txt', { encoding: 'utf8' }) + .pipe(csv.parse({ objectMode: true, headers: true, quote: '"' })) + .on('error', function (e) { + console.error(e); + }); +var tripsPool = createWriteStreams('trips'); +var tripIndex = -1; + +trips.on('data', trip => { + if (tripIndex === -1) { + for (let i in tripsPool) { + tripsPool[i].write(Object.keys(trip)); + } + tripIndex++; + } + + tripsPool[tripIndex].write(Object.values(trip)); + tripIndex = tripIndex < numCPUs - 1 ? tripIndex + 1 : 0; +}); + +trips.on('end', () => { + for (let i in tripsPool) { + tripsPool[i].end(); + } }); + +// Also fragment stop_times.txt +var stopTimes = fs.createReadStream('stop_times.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true, quote: '"' })) + .on('error', function (e) { + console.error(e); + }); + +var connectionsPool = createWriteStreams('connections'); +var connIndex = -1; +var currentTrip = null; +var printedRows = 0; + var connectionRules = stopTimes.pipe(new St2C()); connectionRules.on('data', row => { - printConnectionRule(row); + if (connIndex === -1) { + for (let i in connectionsPool) { + connectionsPool[i].write(Object.keys(row)); + } + } + + if (row['trip_id'] !== currentTrip) { + currentTrip = row['trip_id']; + connIndex = connIndex < numCPUs - 1 ? connIndex + 1 : 0; + } + + connectionsPool[connIndex].write(Object.values(row)) + printedRows++; }); + connectionRules.on('end', () => { - console.error('Converted ' + printedRows + ' stoptimes to connections'); + for (let i in connectionsPool) { + connectionsPool[i].end(); + } + console.error('Converted ' + printedRows + ' stop_times to connections'); }); -var printConnectionHeader = function (row) { - console.log(Object.keys(row).join(',')); -}; - -var printedRows = 0; -var printConnectionRule = function (row) { - if (printedRows === 0) { - printConnectionHeader(row); +function createWriteStreams(name) { + let writers = []; + for (let i = 0; i < numCPUs; i++) { + const stream = csv.format(); + stream.pipe(fs.createWriteStream(name + '_' + i + '.txt', { encoding: 'utf8' })); + writers.push(stream); } - console.log(Object.values(row).join(',')); - printedRows++; -} + return writers; +} diff --git a/lib/Connections2CSV.js b/lib/Connections2CSV.js new file mode 100644 index 0000000..fbe9086 --- /dev/null +++ b/lib/Connections2CSV.js @@ -0,0 +1,33 @@ +const { Transform } = require('stream'); + +class Connections2CSV extends Transform { + constructor(header) { + super({ objectMode: true }); + this._headerStreamed = false; + if(!header) { + this.headerStreamed = true; + } + } + + _transform(connection, encoding, done) { + if (!this.headerStreamed) { + this.headerStreamed = true; + done(null, 'departureStop","departureTime","arrivalStop","arrivalTime","trip","route","headsign"\n'); + } else { + let csv = connection["departureStop"] + ',' + connection["departureTime"].toISOString() + ',' + + connection["arrivalStop"] + ',' + connection["arrivalTime"].toISOString() + ',' + connection["trip"]["trip_id"] + ',' + + connection.trip.route.route_id + ',"' + connection.headsign + '"' + '\n'; + done(null, csv); + } + } + + get headerStreamed() { + return this._headerStreamed; + } + + set headerStreamed(value) { + this._headerStreamed = value; + } +} + +module.exports = Connections2CSV; \ No newline at end of file diff --git a/lib/Connections2JSONLD.js b/lib/Connections2JSONLD.js index f38cd30..f8149b3 100644 --- a/lib/Connections2JSONLD.js +++ b/lib/Connections2JSONLD.js @@ -8,24 +8,13 @@ var Transform = require('stream').Transform, var Connections2JSONLD = function (baseUris, stopsdb, context) { Transform.call(this, { objectMode: true }); - this.context = context || { - "@context": { - "lc": "http://semweb.mmlab.be/ns/linkedconnections#", - "Connection": "http://semweb.mmlab.be/ns/linkedconnections#Connection", - "gtfs": "http://vocab.gtfs.org/terms#", - "departureStop": { - "@type": "@id", - "@id": "http://semweb.mmlab.be/ns/linkedconnections#departureStop" - }, - "arrivalStop": { - "@type": "@id", - "@id": "http://semweb.mmlab.be/ns/linkedconnections#arrivalStop" - }, - "departureTime": "http://semweb.mmlab.be/ns/linkedconnections#departureTime", - "arrivalTime": "http://semweb.mmlab.be/ns/linkedconnections#arrivalTime", - "direction": "gtfs:headsign" - } - }; + this._contextStreamed = false; + this._context = context; + + // Skip context if none provided + if(!this._context) { + this._contextStreamed = true; + } this._uris = new URIStrategy(baseUris, stopsdb); this._count = 0; @@ -33,40 +22,45 @@ var Connections2JSONLD = function (baseUris, stopsdb, context) { util.inherits(Connections2JSONLD, Transform); -Connections2JSONLD.prototype._transform = function (connection, encoding, done) { +Connections2JSONLD.prototype._transform = async function (connection, encoding, done) { try { - var id = this._uris.getId(connection); - const types = ['gtfs:Regular', 'gtfs:NotAvailable', 'gtfs:MustPhone', 'gtfs:MustCoordinateWithDriver']; + if (!this._contextStreamed) { + this._contextStreamed = true; + done(null, this._context); + } else { + var id = this._uris.getId(connection); + const types = ['gtfs:Regular', 'gtfs:NotAvailable', 'gtfs:MustPhone', 'gtfs:MustCoordinateWithDriver']; - var lc = { - "@id": id, - "@type": "Connection", - "departureStop": this._uris.getStopId(connection.departureStop), - "arrivalStop": this._uris.getStopId(connection.arrivalStop), - "departureTime": connection.departureTime.toISOString(), - "arrivalTime": connection.arrivalTime.toISOString(), - "gtfs:trip": this._uris.getTripId(connection), - "gtfs:route": this._uris.getRouteId(connection) - }; + var lc = { + "@id": id, + "@type": "Connection", + "departureStop": await this._uris.getStopId(connection.departureStop), + "arrivalStop": await this._uris.getStopId(connection.arrivalStop), + "departureTime": connection.departureTime, + "arrivalTime": connection.arrivalTime, + "gtfs:trip": this._uris.getTripId(connection), + "gtfs:route": this._uris.getRouteId(connection) + }; - //the headsign is already the result here of earlier checking whether there’s a trip headsign or a route headsign if connection headsign was not set. It can be used reliably - if (connection.headsign) { - lc["direction"] = connection.headsign; - } + //the headsign is already the result here of earlier checking whether there’s a trip headsign or a route headsign if connection headsign was not set. It can be used reliably + if (connection.headsign) { + lc["direction"] = connection.headsign; + } - var pickupType = types[0]; - if (connection['pickup_type'] && connection['pickup_type'] !== null) { - pickupType = types[connection['pickup_type']]; - lc["gtfs:pickupType"] = pickupType; - } + var pickupType = types[0]; + if (connection['pickup_type'] && connection['pickup_type'] !== null) { + pickupType = types[connection['pickup_type']]; + lc["gtfs:pickupType"] = pickupType; + } - var dropOffType = types[0]; - if (connection['drop_off_type'] && connection['drop_off_type'] !== null) { - dropOffType = types[connection['drop_off_type']]; - lc["gtfs:dropOffType"] = dropOffType; - } + var dropOffType = types[0]; + if (connection['drop_off_type'] && connection['drop_off_type'] !== null) { + dropOffType = types[connection['drop_off_type']]; + lc["gtfs:dropOffType"] = dropOffType; + } - done(null, lc); + done(null, lc); + } } catch (err) { console.error(err); done(null, {}); diff --git a/lib/Connections2Triples.js b/lib/Connections2Triples.js index 298762f..c37c67f 100644 --- a/lib/Connections2Triples.js +++ b/lib/Connections2Triples.js @@ -17,7 +17,7 @@ var Connections2Triples = function (baseUris, stopsdb) { util.inherits(Connections2Triples, Transform); -Connections2Triples.prototype._transform = function (connection, encoding, done) { +Connections2Triples.prototype._transform = async function (connection, encoding, done) { var id = this._uris.getId(connection); this.push( quad( @@ -29,13 +29,13 @@ Connections2Triples.prototype._transform = function (connection, encoding, done) quad( namedNode(id), namedNode('http://semweb.mmlab.be/ns/linkedconnections#departureStop'), - namedNode(this._uris.getStopId(connection.departureStop)) + namedNode(await this._uris.getStopId(connection.departureStop)) )); this.push( quad( namedNode(id), namedNode('http://semweb.mmlab.be/ns/linkedconnections#arrivalStop'), - namedNode(this._uris.getStopId(connection.arrivalStop)) + namedNode(await this._uris.getStopId(connection.arrivalStop)) )); this.push( quad( diff --git a/lib/ConnectionsBuilder.js b/lib/ConnectionsBuilder.js index cae7329..ba079a4 100644 --- a/lib/ConnectionsBuilder.js +++ b/lib/ConnectionsBuilder.js @@ -77,7 +77,7 @@ ConnectionsBuilder.prototype.processConnectionRule = function (connectionRule, d if (connectionRule['pickup_type']) { connection['pickup_type'] = connectionRule['pickup_type']; } - + this.push(connection); } done(); @@ -109,9 +109,10 @@ ConnectionsBuilder.prototype._expandTrip = async function(connectionRule) { let route = {}; try { route = await this._routesdb.get(trip['route_id']); - //Hotfix for route long names: usually contain --. However, we are 2018 at the time of writing and can use UTF-8! + // Hotfix for route long names: usually contain --. However, we are 2018 at the time of writing and can use UTF-8! route.route_long_name = route.route_long_name.replace('--','–'); } catch (error) { + console.error(error); console.error('WARNING: somehow a route could not be found: ' + trip.route_id); } trip.route = route; diff --git a/lib/URIStrategy.js b/lib/URIStrategy.js index d9c7fe6..9c4e823 100644 --- a/lib/URIStrategy.js +++ b/lib/URIStrategy.js @@ -33,7 +33,7 @@ var URIStrategy = function (baseUris, stopsdb) { } // Stops index from GTFS (stops.txt) to allow using other params besides 'stop_id' in URI templates - // e.g. 'stop_code' (used by De Lijn!), 'stop_name', stop_lat', 'stop_lon', etc.S + // e.g. 'stop_code' (used by De Lijn!), 'stop_name', stop_lat', 'stop_lon', etc. this._stopsdb = stopsdb; this._stopTemplate = uri_templates(baseUris.stop); @@ -50,11 +50,12 @@ URIStrategy.prototype.getId = function (connection) { return resolveURI(this._connectionTemplate, connection, null, this._resolve); }; -URIStrategy.prototype.getStopId = function (id) { - if(this._stopsdb.has(id)) { - return resolveURI(this._stopTemplate, null, this._stopsdb.get(id), this._resolve); +URIStrategy.prototype.getStopId = async function (id) { + let stopid = await this._stopsdb.get(id); + if(stopid) { + return resolveURI(this._stopTemplate, null, stopid, this._resolve); } else { - throw new Error('Stop ' + id + ' is not defined stops.txt'); + throw new Error('Stop ' + id + ' is not defined in stops.txt'); } }; diff --git a/lib/gtfs2connections.js b/lib/gtfs2connections.js index 591058f..33606ef 100644 --- a/lib/gtfs2connections.js +++ b/lib/gtfs2connections.js @@ -1,10 +1,20 @@ -const csv = require('fast-csv'), - ConnectionsBuilder = require('./ConnectionsBuilder.js'), - Services = require('./services/calendar.js'), - Store = require('./stores/Store.js'), - { AsyncIterator } = require('asynciterator'), - StreamIterator = require('./StreamIterator.js'), - fs = require('fs'); +const { Worker, isMainThread, parentPort, workerData } = require('worker_threads'); +const fs = require('fs'); +const { AsyncIterator } = require('asynciterator'); +const csv = require('fast-csv'); +const N3 = require('n3'); +const Store = require('./stores/Store'); +const Services = require('./services/calendar'); +const StreamIterator = require('./StreamIterator'); +const ConnectionsBuilder = require('./ConnectionsBuilder'); +const Connections2JSONLD = require('./Connections2JSONLD'); +const Connections2CSV = require('./Connections2CSV'); +const Connections2Mongo = require('./Connections2Mongo'); +const Connections2Triples = require('./Connections2Triples'); +const jsonldstream = require('jsonld-stream'); +const { exec } = require('child_process'); +const del = require('del'); +const numCPUs = require('os').cpus().length; var Mapper = function (options) { this._options = options; @@ -15,53 +25,115 @@ var Mapper = function (options) { /** * Returns a resultStream for connections -* Step 1: Convert calendar_dates.txt and calendar.txt to service ids mapped to a long list of dates -* Step 2: Pipe these services towards a leveldb: we want to use them later. -* Step 3: also index routes.txt and trips.txt in a leveldb on key trip_id -* Step 4: create a stream of connection rules from stop_times.txt -* Step 5: pipe this stream to something that expands everything into connections and returns this stream. +* Step 1: Read stops.txt and routes.txt and, convert calendar_dates.txt and calendar.txt to service ids mapped to a long list of dates. +* Step 2: Store stops, routes and services into a levelDB or a in-memory Map. +* Step 3: Use Node.js worker threads to process the connection rules in parallel. +* Step 4: Merge the files created in parallel and return the file path. * Caveat: coding this with numerous callbacks and streams, makes this code not chronologically ordered. */ -Mapper.prototype.resultStream = function (path, done) { - var stops = fs.createReadStream(path + '/stops.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error(e); - }); +Mapper.prototype.resultStream = function (path, output, done) { + let t0 = new Date(); - var routes = fs.createReadStream(path + '/routes.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error(e); - }); + // Step 1: Read all the required GTFS files in a streamed-fashion + var stops = fs.createReadStream(path + '/stops.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .on('error', function (e) { + console.error(e); + }); - var tripsIterator = new StreamIterator(fs.createReadStream(path + '/trips.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error(e); - })); + var routes = fs.createReadStream(path + '/routes.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .on('error', function (e) { + console.error(e); + }); - var calendarDates = fs.createReadStream(path + '/calendar_dates.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error(e); - }); + var calendarDates = fs.createReadStream(path + '/calendar_dates.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .on('error', function (e) { + console.error(e); + }); - var services = fs.createReadStream(path + '/calendar.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).pipe(new Services(calendarDates, this._options)).on('error', function (e) { - console.error(e); - }); - //Preparations for step 4 - //Step 2 & 3: store in leveldb in 3 hidden directories, or in memory, depending on the options - var stopsdb = new Map(); - var routesdb = Store(path + '/.routes', this._options.store); - var servicesdb = Store(path + '/.services', this._options.store); - var count = 0; + var services = fs.createReadStream(path + '/calendar.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .pipe(new Services(calendarDates, this._options)) + .on('error', function (e) { + console.error(e); + }); + + // Step 2: store in Keyv in hidden directories, or in memory, depending on the options var options = this._options; + var stopsdb = Store(output + '/.stops', options.store); + var routesdb = Store(output + '/.routes', options.store); + var servicesdb = Store(output + '/.services', options.store); + var count = 0; + + // Step 3: Create connections in parallel using worker threads var finished = function () { count++; - //wait for the 2 streams to finish (services and routes) to write to the stores + // Wait for the 3 streams to finish (services, routes and stops) to write to the stores if (count === 3) { - console.error("Indexing services and routes successful!"); - //Step 4 and 5: let's create our connections! - var connectionRules = fs.createReadStream(path + '/connections.txt', { encoding: 'utf8', objectMode: true }).pipe(csv.parse({ objectMode: true, headers: true })).on('error', function (e) { - console.error('Hint: Did you run gtfs2lc-sort?'); - console.error(e); - }); - let connectionsBuilder = new ConnectionsBuilder(tripsIterator, servicesdb, routesdb, options); - let connectionsStream = connectionRules.pipe(connectionsBuilder); - done(connectionsStream, stopsdb); + console.error("Indexing stops, services and routes successful!"); + + let w = 0; + + // Create as many worker threads as there are available CPUs + for (let i = 0; i < numCPUs; i++) { + const worker = new Worker(__filename, { + workerData: { + instance: i, + path: path, + output: output, + routesdb: options.store === 'MemStore' ? routesdb : path + '/.routes', + servicesdb: options.store === 'MemStore' ? servicesdb : path + '/.services', + stopsdb: options.store === 'MemStore' ? stopsdb : path + '/.stops', + options: options + } + }); + + worker.on('message', () => { + w++; + if (w === numCPUs) { + // Merge all the created files into one + let format = options['format']; + let ext = null; + if (!format || ['json', 'mongo', 'jsonld', 'mongold'].indexOf(format) >= 0) { + ext = 'json'; + } else if (format === 'csv') { + ext = 'csv'; + } else if (format = 'turtle') { + ext = 'ttl'; + } else if (format === 'ntriples') { + ext = 'n3'; + } + + exec('cat raw_* > linkedConnections.' + ext, { cwd: output }, err => { + if (err) { + throw err; + } + let t1 = new Date(); + console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms'); + del([ + path + '/connections_*', + path + '/trips_*', + output + '/.stops', + output + '/.routes', + output + '/.services', + output + '/raw_*' + ], + { force: true } + ).then(function () { + done(output + '/linkedConnections.' + ext); + }); + }); + } + }).on('error', err => { + console.error(err); + }).on('exit', (code) => { + if (code !== 0) { + console.error(new Error(`Worker stopped with exit code ${code}`)); + } + }); + } } }; @@ -71,7 +143,14 @@ Mapper.prototype.resultStream = function (path, done) { servicesIterator.transform((service, doneService) => { if (service['service_id']) { - servicesdb.put(service['service_id'], service['dates'], doneService); + if (servicesdb instanceof Map) { + servicesdb.set(service['service_id'], service['dates']); + doneService(); + } else { + servicesdb.set(service['service_id'], service['dates']).then(function () { + doneService(); + }); + } } else { doneService(); } @@ -82,7 +161,14 @@ Mapper.prototype.resultStream = function (path, done) { routesIterator.transform((route, doneRoute) => { if (route['route_id']) { - routesdb.put(route['route_id'], route, doneRoute); + if (routesdb instanceof Map) { + routesdb.set(route['route_id'], route); + doneRoute(); + } else { + routesdb.set(route['route_id'], route).then(function () { + doneRoute(); + }); + } } else { doneRoute(); } @@ -93,13 +179,126 @@ Mapper.prototype.resultStream = function (path, done) { stopsIterator.transform((stop, doneStop) => { if (stop['stop_id']) { - stopsdb.set(stop['stop_id'], stop); - } - doneStop(); + if (stopsdb instanceof Map) { + stopsdb.set(stop['stop_id'], stop); + doneStop(); + } else { + stopsdb.set(stop['stop_id'], stop).then(function () { + doneStop(); + }); + } + } }).on('data', () => { }).on('error', function (e) { console.error(e); }).on('end', finished); }; +// Code executed only on a Worker Thread +if (!isMainThread) { + console.error('Created worker thread ' + workerData['instance']); + // Read the connections file created by the gtfs2lc-sort script + var connectionRules = fs.createReadStream(workerData['path'] + '/connections_' + workerData['instance'] + '.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .on('error', function (e) { + console.error('Hint: Did you run gtfs2lc-sort?'); + console.error(e); + }); + // Read the corresponding trips information + var tripsIterator = new StreamIterator(fs.createReadStream(workerData['path'] + '/trips_' + workerData['instance'] + '.txt', { encoding: 'utf8', objectMode: true }) + .pipe(csv.parse({ objectMode: true, headers: true })) + .on('error', function (e) { + console.error(e); + })); + + let routesdb = null; + let servicesdb = null; + let stopsdb = null; + + if (workerData['options']['store'] === 'KeyvStore') { + // Rebuild the KeyvStore objects + routesdb = Store(workerData['routesdb'], 'KeyvStore'); + servicesdb = Store(workerData['servicesdb'], 'KeyvStore'); + stopsdb = Store(workerData['stopsdb'], 'KeyvStore'); + } else { + routesdb = workerData['routesdb']; + servicesdb = workerData['servicesdb']; + stopsdb = workerData['stopsdb']; + } + + // Build the Connection objects! + let connectionsBuilder = new ConnectionsBuilder(tripsIterator, servicesdb, routesdb, workerData['options']); + let connectionStream = connectionRules.pipe(connectionsBuilder); + + // Now, proceed to parse the connections according to the requested format + let format = workerData['options']['format']; + + if (!format || ['json', 'mongo'].indexOf(format) >= 0) { + if (format === 'mongo') { + connectionStream = connectionStream.pipe(new Connections2Mongo()); + } + + connectionStream = connectionStream.pipe(new jsonldstream.Serializer()) + .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.json')); + } else if (['jsonld', 'mongold'].indexOf(format) >= 0) { + let context = undefined; + + // Only include the context for the first instance + if (workerData['instance'] === 0) { + context = { + '@context': { + lc: 'http://semweb.mmlab.be/ns/linkedconnections#', + gtfs: 'http://vocab.gtfs.org/terms#', + xsd: 'http://www.w3.org/2001/XMLSchema#', + trip: { '@type': '@id', '@id': 'gtfs:trip' }, + Connection: 'lc:Connection', + CancelledConnection: 'lc:CancelledConnection', + departureTime: { '@type': 'xsd:dateTime', '@id': 'lc:departureTime' }, + departureStop: { '@type': '@id', '@id': 'lc:departureStop' }, + arrivalStop: { '@type': '@id', '@id': 'lc:arrivalStop' }, + arrivalTime: { '@type': 'xsd:dateTime', '@id': 'lc:arrivalTime' }, + } + }; + } + // Convert json object stream to jsonld stream + connectionStream = connectionStream.pipe(new Connections2JSONLD(workerData['options']['baseUris'], stopsdb, context)); + + if (format === 'mongold') { + connectionStream = connectionStream.pipe(new Connections2Mongo()); + } + + connectionStream = connectionStream.pipe(new jsonldstream.Serializer()) + .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.json')); + } else if (format === 'csv') { + let header = false; + if (workerData['instance'] === 0) { + header = true; + } + connectionStream = connectionStream.pipe(new Connections2CSV(header)) + .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.csv')); + } else if (format === 'turtle') { + let prefixes = undefined; + + if (workerData['instance'] === 0) { + prefixes = { + lc: 'http://semweb.mmlab.be/ns/linkedconnections#', + gtfs: 'http://vocab.gtfs.org/terms#', + xsd: 'http://www.w3.org/2001/XMLSchema#' + }; + } + connectionStream = connectionStream.pipe(new Connections2Triples(workerData['options']['baseUris'], stopsdb)) + .pipe(new N3.StreamWriter({ prefixes: prefixes })) + .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.ttl')); + } else if (format === 'ntriples') { + connectionStream = connectionStream.pipe(new Connections2Triples(workerData['options']['baseUris'], stopsdb)) + .pipe(new N3.StreamWriter({ format: 'N-Triples' })) + .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.n3')); + } + + connectionStream.on('finish', () => { + parentPort.postMessage('done'); + }); + +} + module.exports = Mapper; diff --git a/lib/stores/LevelStore.js b/lib/stores/LevelStore.js deleted file mode 100644 index 61d6a2a..0000000 --- a/lib/stores/LevelStore.js +++ /dev/null @@ -1,36 +0,0 @@ -var level = require('level'); - -var LevelStore = function (name) { - this.name = name; - this._store = level(name); -}; - -LevelStore.prototype.get = function (key, cb) { - return new Promise((resolve, reject) => { - this._store.get(key, function (error, object) { - if (!error) { - if(cb) - cb(null, JSON.parse(object)); - resolve(JSON.parse(object)); - } else { - cb(error); - reject(error); - } - }); - }); -}; - -LevelStore.prototype.put = function (key, value, cb) { - return new Promise((resolve, reject) => { - this._store.put(key, value, {valueEncoding: 'json'}, (error) => { - if (cb) - cb(error); - if (error) - reject(error); - else - resolve(); - }); - }); -}; - -module.exports = LevelStore; diff --git a/lib/stores/MemStore.js b/lib/stores/MemStore.js deleted file mode 100644 index 06f958c..0000000 --- a/lib/stores/MemStore.js +++ /dev/null @@ -1,37 +0,0 @@ -var MemStore = function (name) { - this.name = name; - this._store = {}; -}; - -MemStore.prototype.get = function (key, cb) { - return new Promise((resolve, reject) => { - if (this._store[key]) { - if (cb) - cb(null, this._store[key]); - resolve(this._store[key]); - } else { - if (cb) - cb(key + ' not found in store'); - reject(key + ' not found in store'); - } - }); -}; - -MemStore.prototype.put = function (key, value, cb) { - return new Promise((resolve, reject) => { - if (this._store[key]) { - //don't add it again, but give a warning - console.error('WARNING: ' + key + ' already exists. Throwing away this value: ' + value); - if (cb) - cb(); - resolve(); - } else { - this._store[key] = value; - if (cb) - cb(); - resolve(); - } - }); -}; - -module.exports = MemStore; diff --git a/lib/stores/Store.js b/lib/stores/Store.js index 09f559b..68b0f9b 100644 --- a/lib/stores/Store.js +++ b/lib/stores/Store.js @@ -1,13 +1,16 @@ -var MemStore = require('./MemStore'), - util = require('util'), - LevelStore = require('./LevelStore'); +const Keyv = require('keyv'); +const KeyvFile = require('keyv-file'); module.exports = function (config, type) { var store; if (type === 'MemStore') { - store = new MemStore(config); - } else if (type === 'LevelStore') { - store = new LevelStore(config); + store = new Map(); + } else { + store = new Keyv({ + store: new KeyvFile({ + filename: config + }) + }); } return store; }; diff --git a/package-lock.json b/package-lock.json index a57fdb3..ebcc41b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -396,6 +396,29 @@ "@types/yargs": "^12.0.9" } }, + "@nodelib/fs.scandir": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.1.tgz", + "integrity": "sha512-NT/skIZjgotDSiXs0WqYhgcuBKhUMgfekCmCGtkUAiLqZdOnrdjmZr9wRl3ll64J9NF79uZ4fk16Dx0yMc/Xbg==", + "requires": { + "@nodelib/fs.stat": "2.0.1", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.1.tgz", + "integrity": "sha512-+RqhBlLn6YRBGOIoVYthsG0J9dfpO79eJyN7BYBkZJtfqrBwf2KK+rD/M/yjZR6WBmIhAgOV7S60eCgaSWtbFw==" + }, + "@nodelib/fs.walk": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.2.tgz", + "integrity": "sha512-J/DR3+W12uCzAJkw7niXDcqcKBg6+5G5Q/ZpThpGNzAUz70eOR6RV4XnnSN01qHZiVl0eavoxJsBypQoKsV2QQ==", + "requires": { + "@nodelib/fs.scandir": "2.1.1", + "fastq": "^1.6.0" + } + }, "@types/babel__core": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.2.tgz", @@ -437,6 +460,21 @@ "@babel/types": "^7.3.0" } }, + "@types/events": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz", + "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==" + }, + "@types/glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", + "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==", + "requires": { + "@types/events": "*", + "@types/minimatch": "*", + "@types/node": "*" + } + }, "@types/istanbul-lib-coverage": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz", @@ -467,6 +505,11 @@ "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.134.tgz", "integrity": "sha512-2/O0khFUCFeDlbi7sZ7ZFRCcT812fAeOLm7Ev4KbwASkZ575TDrDcY7YyaoHdTOzKcNbfiwLYZqPmoC4wadrsw==" }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" + }, "@types/node": { "version": "12.0.10", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz", @@ -499,15 +542,6 @@ "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==", "dev": true }, - "abstract-leveldown": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.0.3.tgz", - "integrity": "sha512-jzewKKpZbaYUa6HTThnrl+GrJhzjEAeuc7hTVpZdzg7kupXZFoqQDFwyOwLNbmJKJlmzw8yiipMPkDiuKkT06Q==", - "requires": { - "level-concat-iterator": "~2.0.0", - "xtend": "~4.0.0" - } - }, "acorn": { "version": "5.7.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", @@ -613,6 +647,11 @@ "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==" + }, "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", @@ -724,8 +763,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base": { "version": "0.11.2", @@ -794,7 +832,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1030,8 +1067,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "convert-source-map": { "version": "1.6.0", @@ -1140,6 +1176,14 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz", "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==" }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -1158,15 +1202,6 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, - "deferred-leveldown": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.0.1.tgz", - "integrity": "sha512-BXohsvTedWOLkj2n/TY+yqVlrCWa2Zs8LSxh3uCAgFOru7/pjxKyZAexGa1j83BaKloER4PqUyQ9rGPJLt9bqA==", - "requires": { - "abstract-leveldown": "~6.0.0", - "inherits": "^2.0.3" - } - }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -1217,6 +1252,18 @@ } } }, + "del": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/del/-/del-5.0.0.tgz", + "integrity": "sha512-TfU3nUY0WDIhN18eq+pgpbLY9AfL5RfiE9czKaTSolc6aK7qASXfDErvYgjV1UqCR4sNXDoxO0/idPmhDUt2Sg==", + "requires": { + "globby": "^10.0.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "rimraf": "^2.6.3" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1234,6 +1281,21 @@ "integrity": "sha512-xLqpez+Zj9GKSnPWS0WZw1igGocZ+uua8+y+5dDNTT934N3QuY1sp2LkHzwiaYQGz60hMq0pjAshdeXm5VUOEw==", "dev": true }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "requires": { + "path-type": "^4.0.0" + }, + "dependencies": { + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" + } + } + }, "domexception": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", @@ -1252,17 +1314,6 @@ "safer-buffer": "^2.1.0" } }, - "encoding-down": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.0.2.tgz", - "integrity": "sha512-oAEANslmNb64AF4kvHXjTxB7KecwD7X0qf8MffMfhpjP6gjGcnCTOkRgps/1yUNeR4Bhe6ckN6aAzZz+RIYgTw==", - "requires": { - "abstract-leveldown": "^6.0.0", - "inherits": "^2.0.3", - "level-codec": "^9.0.0", - "level-errors": "^2.0.0" - } - }, "end-of-stream": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", @@ -1272,14 +1323,6 @@ "once": "^1.4.0" } }, - "errno": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", - "requires": { - "prr": "~1.0.1" - } - }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -1566,10 +1609,58 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" }, - "fast-future": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/fast-future/-/fast-future-1.0.2.tgz", - "integrity": "sha1-hDWpqqAteSSNF9cE52JZMB2ZKAo=" + "fast-glob": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.0.4.tgz", + "integrity": "sha512-wkIbV6qg37xTJwqSsdnIphL1e+LaGz4AIQqr00mIubMaEhv1/HEmJ0uuCGZRNRUkZZmOB5mJKO0ZUTVq+SxMQg==", + "requires": { + "@nodelib/fs.stat": "^2.0.1", + "@nodelib/fs.walk": "^1.2.1", + "glob-parent": "^5.0.0", + "is-glob": "^4.0.1", + "merge2": "^1.2.3", + "micromatch": "^4.0.2" + }, + "dependencies": { + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "requires": { + "fill-range": "^7.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + }, + "micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.0.5" + } + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "requires": { + "is-number": "^7.0.0" + } + } + } }, "fast-json-stable-stringify": { "version": "2.0.0", @@ -1582,6 +1673,14 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fastq": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.6.0.tgz", + "integrity": "sha512-jmxqQ3Z/nXoeyDmWAzF9kH1aGZSis6e/SbfPmJpUnyZ0ogr6iscHQaml4wsEepEWSdtmpy+eVXmCRIMpxaXqOA==", + "requires": { + "reusify": "^1.0.0" + } + }, "fb-watchman": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.0.tgz", @@ -1663,11 +1762,20 @@ "map-cache": "^0.2.2" } }, + "fs-extra": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz", + "integrity": "sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "1.2.9", @@ -2250,7 +2358,6 @@ "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2260,17 +2367,46 @@ "path-is-absolute": "^1.0.0" } }, + "glob-parent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.0.0.tgz", + "integrity": "sha512-Z2RwiujPRGluePM6j699ktJYxmPpJKCfpGA13jz2hmFZC7gKetzrWvg5KN3+OsIFmydGyZ1AVwERCq1w/ZZwRg==", + "requires": { + "is-glob": "^4.0.1" + } + }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, + "globby": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.1.tgz", + "integrity": "sha512-sSs4inE1FB2YQiymcmTv6NWENryABjUNPeWhOvmn4SjtKybglsyPZxFB3U1/+L1bYi0rNZDqCLlHyLYDl1Pq5A==", + "requires": { + "@types/glob": "^7.1.1", + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.0.3", + "glob": "^7.1.3", + "ignore": "^5.1.1", + "merge2": "^1.2.3", + "slash": "^3.0.0" + }, + "dependencies": { + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==" + } + } + }, "graceful-fs": { "version": "4.1.15", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", - "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", - "dev": true + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" }, "growl": { "version": "1.10.5", @@ -2403,10 +2539,10 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "immediate": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.2.3.tgz", - "integrity": "sha1-0UD6j2FGWb1lQSMwl92qwlzdmRw=" + "ignore": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.2.tgz", + "integrity": "sha512-vdqWBp7MyzdmHkkRWV5nY+PfGRbYbahfuvsBCh277tq+w9zyNi7h5CYJCK0kmzti9kU+O/cB7sE8HvKv6aXAKQ==" }, "import-local": { "version": "2.0.0", @@ -2428,7 +2564,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -2558,6 +2693,11 @@ "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", "dev": true }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=" + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", @@ -2570,6 +2710,14 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "requires": { + "is-extglob": "^2.1.1" + } + }, "is-number": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", @@ -2596,6 +2744,27 @@ } } }, + "is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==" + }, + "is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "requires": { + "is-path-inside": "^2.1.0" + } + }, + "is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "requires": { + "path-is-inside": "^1.0.2" + } + }, "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", @@ -3316,6 +3485,11 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -3354,6 +3528,14 @@ } } }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, "jsonld": { "version": "0.4.12", "resolved": "https://registry.npmjs.org/jsonld/-/jsonld-0.4.12.tgz", @@ -3399,6 +3581,24 @@ "verror": "1.10.0" } }, + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "requires": { + "json-buffer": "3.0.0" + } + }, + "keyv-file": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/keyv-file/-/keyv-file-0.1.13.tgz", + "integrity": "sha512-dXjzhVgmCxPDzr1BoYBn9bmHdAArmQtc0uSf4Cido1+CQydbfYyliwMybkSfcELu4oOwDAJ1ui1EKn3H9jgDmg==", + "requires": { + "debug": "^4.1.1", + "fs-extra": "^4.0.1", + "tslib": "^1.9.3" + } + }, "kind-of": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", @@ -3432,88 +3632,6 @@ "integrity": "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==", "dev": true }, - "level": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/level/-/level-5.0.1.tgz", - "integrity": "sha512-wcak5OQeA4rURGacqS62R/xNHjCYnJSQDBOlm4KNUGJVE9bWv2B04TclqReYejN+oD65PzD4FsqeWoI5wNC5Lg==", - "requires": { - "level-js": "^4.0.0", - "level-packager": "^5.0.0", - "leveldown": "^5.0.0", - "opencollective-postinstall": "^2.0.0" - } - }, - "level-codec": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.1.tgz", - "integrity": "sha512-ajFP0kJ+nyq4i6kptSM+mAvJKLOg1X5FiFPtLG9M5gCEZyBmgDi3FkDrvlMkEzrUn1cWxtvVmrvoS4ASyO/q+Q==" - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", - "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", - "requires": { - "errno": "~0.1.1" - } - }, - "level-iterator-stream": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.1.tgz", - "integrity": "sha512-pSZWqXK6/yHQkZKCHrR59nKpU5iqorKM22C/BOHTb/cwNQ2EOZG+bovmFFGcOgaBoF3KxqJEI27YwewhJQTzsw==", - "requires": { - "inherits": "^2.0.1", - "readable-stream": "^3.0.2", - "xtend": "^4.0.0" - } - }, - "level-js": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/level-js/-/level-js-4.0.1.tgz", - "integrity": "sha512-m5JRIyHZn5VnCCFeRegJkn5bQd3MJK5qZX12zg3Oivc8+BUIS2yFS6ANMMeHX2ieGxucNvEn6/ZnyjmZQLLUWw==", - "requires": { - "abstract-leveldown": "~6.0.1", - "immediate": "~3.2.3", - "inherits": "^2.0.3", - "ltgt": "^2.1.2", - "typedarray-to-buffer": "~3.1.5" - } - }, - "level-packager": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.0.1.tgz", - "integrity": "sha512-tigB8g7xnFE5es2d/OmGJvcJC9S+FQfJnnULSLbPr53/ABPIaCarCxofkwdBhnJxjLZCNvj/Je6luFj/XeuaUQ==", - "requires": { - "encoding-down": "^6.0.0", - "levelup": "^4.0.0" - } - }, - "leveldown": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.1.0.tgz", - "integrity": "sha512-+Ws1IDVSbJRJl/cP/JlapGSmt8xPSvZQpTlYGpyaMQiXtlx/fdMX30K2aHiTLxhLcnAg74D8odBG1Fk12/8SRA==", - "requires": { - "abstract-leveldown": "~6.0.3", - "fast-future": "~1.0.2", - "napi-macros": "~1.8.1", - "node-gyp-build": "~4.1.0" - } - }, - "levelup": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.0.1.tgz", - "integrity": "sha512-l7KXOkINXHgNqmz0v9bxvRnMCUG4gmShFrzFSZXXhcqFnfvKAW8NerVsTICpZtVhGOMAmhY6JsVoVh/tUPBmdg==", - "requires": { - "deferred-leveldown": "~5.0.0", - "level-errors": "~2.0.0", - "level-iterator-stream": "~4.0.0", - "xtend": "~4.0.0" - } - }, "leven": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", @@ -3553,9 +3671,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "lodash.sortby": { "version": "4.7.0", @@ -3578,11 +3696,6 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, - "ltgt": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", - "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" - }, "make-dir": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", @@ -3680,6 +3793,11 @@ } } }, + "merge2": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.2.3.tgz", + "integrity": "sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA==" + }, "micromatch": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", @@ -3724,7 +3842,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -3736,9 +3853,9 @@ "dev": true }, "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", "dev": true, "requires": { "for-in": "^1.0.2", @@ -3768,8 +3885,7 @@ "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", - "dev": true + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "n3": { "version": "1.1.1", @@ -3802,11 +3918,6 @@ "to-regex": "^3.0.1" } }, - "napi-macros": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-1.8.2.tgz", - "integrity": "sha512-Tr0DNY4RzTaBG2W2m3l7ZtFuJChTH6VZhXVhkGGjF/4cZTt+i8GcM9ozD+30Lmr4mDoZ5Xx34t2o4GJqYWDGcg==" - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3825,11 +3936,6 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, - "node-gyp-build": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.0.tgz", - "integrity": "sha512-rGLv++nK20BG8gc0MzzcYe1Nl3p3mtwJ74Q2QD0HTEDKZ6NvOFSelY6s2QBPWIHRR8h7hpad0LiwajfClBJfNg==" - }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -3977,16 +4083,10 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1" } }, - "opencollective-postinstall": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz", - "integrity": "sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==" - }, "optimist": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", @@ -4075,6 +4175,11 @@ "p-limit": "^2.0.0" } }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, "p-reduce": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-reduce/-/p-reduce-1.0.0.tgz", @@ -4118,8 +4223,12 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" }, "path-key": { "version": "2.0.1", @@ -4147,6 +4256,11 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "picomatch": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.0.7.tgz", + "integrity": "sha512-oLHIdio3tZ0qH76NybpeneBhYVj0QFTfXEFTc/B3zKQspYfYYkWYgFsmzo+4kvId/bQRcNkVeguI3y+CD22BtA==" + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", @@ -4230,11 +4344,6 @@ "sisteransi": "^1.0.0" } }, - "prr": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" - }, "psl": { "version": "1.1.29", "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", @@ -4287,16 +4396,6 @@ "read-pkg": "^3.0.0" } }, - "readable-stream": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.3.0.tgz", - "integrity": "sha512-EsI+s3k3XsW+fU8fQACLN59ky34AZ14LoeVZpYwmZvldCFo0r0gnelwF2TcMjLor/BTL5aDJVBMkss0dthToPw==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, "realpath-native": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-1.1.0.tgz", @@ -4436,11 +4535,15 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" + }, "rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "dev": true, "requires": { "glob": "^7.1.3" } @@ -4451,6 +4554,11 @@ "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==", "dev": true }, + "run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==" + }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -4514,9 +4622,9 @@ "dev": true }, "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", "dev": true, "requires": { "extend-shallow": "^2.0.1", @@ -4860,14 +4968,6 @@ "strip-ansi": "^4.0.0" } }, - "string_decoder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", - "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -5010,6 +5110,11 @@ "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", "dev": true }, + "tslib": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -5032,14 +5137,6 @@ "prelude-ls": "~1.1.2" } }, - "typedarray-to-buffer": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", - "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", - "requires": { - "is-typedarray": "^1.0.0" - } - }, "uglify-js": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", @@ -5052,40 +5149,22 @@ } }, "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", "dev": true, "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } + "set-value": "^2.0.1" } }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", @@ -5146,7 +5225,8 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true }, "util.promisify": { "version": "1.0.0", @@ -5304,8 +5384,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write-file-atomic": { "version": "2.4.1", @@ -5338,11 +5417,6 @@ "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.19.tgz", "integrity": "sha1-Yx/Ad3bv2EEYvyUXGzftTQdaCrw=" }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", diff --git a/package.json b/package.json index 324b2a7..c0ccce0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "linkedconnections-joinandsort": "./bin/linkedconnections-sort.sh" }, "scripts": { - "test": "rm -rf test/sample-feed/ ; cp -r test/sample-feed-test/ test/sample-feed/ ; ./bin/gtfs2lc-sort.sh test/sample-feed ; ./bin/gtfs2lc.js -f jsonld test/sample-feed > test/sample-feed/connections-notjoined.nldjsonld ; ./bin/linkedconnections-sort.sh test/sample-feed/connections-notjoined.nldjsonld > test/sample-feed/connections.nldjsonld ; jest", + "test": "rm -rf test/sample-feed/ ; cp -r test/sample-feed-test/ test/sample-feed/ ; ./bin/gtfs2lc-sort.sh test/sample-feed ; ./bin/gtfs2lc.js -s -f jsonld test/sample-feed > test/sample-feed/connections-notjoined.nldjsonld ; ./bin/linkedconnections-sort.sh test/sample-feed/connections-notjoined.nldjsonld > test/sample-feed/connections.nldjsonld ; jest", "coveralls": "jest --coverage && cat ./coverage/lcov.info | coveralls" }, "repository": { @@ -31,9 +31,11 @@ "asynciterator": "^2.0.1", "commander": "^2.20.0", "date-fns": "^1.30.1", + "del": "^5.0.0", "fast-csv": "^3.0.1", "jsonld-stream": "^1.0.4", - "level": "^5.0.1", + "keyv": "^3.1.0", + "keyv-file": "^0.1.13", "n3": "^1.1.1", "uri-templates": "^0.2.0" }, diff --git a/test/resultStream.test.js b/test/resultStream.test.js index d96bed4..3a6122e 100644 --- a/test/resultStream.test.js +++ b/test/resultStream.test.js @@ -1,67 +1,67 @@ const gtfs2lc = require('../lib/gtfs2lc.js'); const assert = require('assert'); -const N3 = require('n3'); +const { exec } = require('child_process'); +const fs = require('fs'); +const util = require('util'); -jest.setTimeout(5000); +const readFile = util.promisify(fs.readFile); + +jest.setTimeout(60000); describe('Testing whether result contains certain objects (regression tests)', () => { - var lcstreamToArray = function (options) { - if (!options) - options = {}; + var doGtfsSort = () => { return new Promise((resolve, reject) => { - var mapper = new gtfs2lc.Connections(options); - mapper.resultStream('./test/sample-feed', (stream, stopsdb) => { - - if (options.format && options.format === 'jsonld') { - stream = stream.pipe(new gtfs2lc.Connections2JSONLD(null, stopsdb, null)); - } - - if (options.format && options.format === "turtle") { - stream = stream.pipe(new gtfs2lc.Connections2Triples({}, stopsdb)) - .pipe(new N3.StreamWriter( - { - prefixes: { - lc: 'http://semweb.mmlab.be/ns/linkedconnections#', - gtfs: 'http://vocab.gtfs.org/terms#', - xsd: 'http://www.w3.org/2001/XMLSchema#' - } - })); + exec(`./bin/gtfs2lc-sort.sh test/sample-feed`, (err, stdout, stderr) => { + if (err) { + reject(stderr); + } else { + resolve(); } + }); + }); + }; - var connections = []; - stream.on('data', connection => { - connections.push(connection); - }); - stream.on('error', (error) => { - reject(error); - }); - stream.on('end', () => { - resolve(connections); + var lcstreamToArray = (options, file) => { + return new Promise((resolve, reject) => { + exec(`./bin/gtfs2lc.js -s -f ${options['format']} -S ${options['store']} test/sample-feed > test/sample-feed/${file}`, + async (err, stdout, stderr) => { + if (err) { + reject(stderr); + } else { + resolve((await readFile(`./test/sample-feed/${file}`, 'utf8')).split('\n')) + } }); - }); }); }; var connections; //This will be the first element when sorted correctly it('Stream should contain a first connection with arrivalStop AMV', async () => { - connections = await lcstreamToArray(); - assert.equal(connections[0]['arrivalStop'], 'AMV'); + try { + await doGtfsSort(); + connections = await lcstreamToArray({}, 'result.json'); + assert.equal(JSON.parse(connections[0])['arrivalStop'], 'AMV'); + } catch (err) { + console.error(err); + } }); - it('JSON-LD Stream should contain Connections and use LevelStore for data storage', async () => { + it('JSON-LD Stream should contain Connections and use KeyvStore for data storage', async () => { + await doGtfsSort(); var triples = await lcstreamToArray({ format: 'jsonld', - store: 'LevelStore' - }); - assert.equal(triples[0]['@type'], 'Connection'); + store: 'KeyvStore' + }, 'result.jsonld'); + assert.equal(JSON.parse(triples[1])['@type'], 'Connection'); }); it('RDF Stream should contain Connections in turtle format', async () => { + await doGtfsSort(); var triples = await lcstreamToArray({ - format: 'turtle' - }); + format: 'turtle', + store: 'MemStore' + }, 'turtle.ttl'); assert.equal(triples[4].includes('a lc:Connection'), true); }); }); From 0bc2acbb72e9ef930b92fa4dda51d4d8fda10db7 Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Fri, 14 Feb 2020 19:04:31 +0100 Subject: [PATCH 2/6] Fix file joining + add some testing --- lib/Connections2CSV.js | 2 +- lib/gtfs2connections.js | 64 +++++++++++++++++++++------------ package.json | 4 +-- test/dataConversion.test.js | 70 +++++++++++++++++++++++++++++++++++++ test/resultStream.test.js | 5 --- 5 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 test/dataConversion.test.js diff --git a/lib/Connections2CSV.js b/lib/Connections2CSV.js index fbe9086..70001a1 100644 --- a/lib/Connections2CSV.js +++ b/lib/Connections2CSV.js @@ -12,7 +12,7 @@ class Connections2CSV extends Transform { _transform(connection, encoding, done) { if (!this.headerStreamed) { this.headerStreamed = true; - done(null, 'departureStop","departureTime","arrivalStop","arrivalTime","trip","route","headsign"\n'); + done(null, '"departureStop","departureTime","arrivalStop","arrivalTime","trip","route","headsign"\n'); } else { let csv = connection["departureStop"] + ',' + connection["departureTime"].toISOString() + ',' + connection["arrivalStop"] + ',' + connection["arrivalTime"].toISOString() + ',' + connection["trip"]["trip_id"] + ',' diff --git a/lib/gtfs2connections.js b/lib/gtfs2connections.js index 33606ef..2e4cf26 100644 --- a/lib/gtfs2connections.js +++ b/lib/gtfs2connections.js @@ -1,5 +1,6 @@ const { Worker, isMainThread, parentPort, workerData } = require('worker_threads'); const fs = require('fs'); +const util = require('util'); const { AsyncIterator } = require('asynciterator'); const csv = require('fast-csv'); const N3 = require('n3'); @@ -12,10 +13,14 @@ const Connections2CSV = require('./Connections2CSV'); const Connections2Mongo = require('./Connections2Mongo'); const Connections2Triples = require('./Connections2Triples'); const jsonldstream = require('jsonld-stream'); -const { exec } = require('child_process'); +const cp = require('child_process'); const del = require('del'); const numCPUs = require('os').cpus().length; +const readdir = util.promisify(fs.readdir); +const appendFile = util.promisify(fs.appendFile); +const exec = util.promisify(cp.exec); + var Mapper = function (options) { this._options = options; if (!this._options.store) { @@ -90,29 +95,29 @@ Mapper.prototype.resultStream = function (path, output, done) { } }); - worker.on('message', () => { + worker.on('message', async () => { w++; if (w === numCPUs) { // Merge all the created files into one let format = options['format']; let ext = null; if (!format || ['json', 'mongo', 'jsonld', 'mongold'].indexOf(format) >= 0) { + await appendLineBreaks(); ext = 'json'; } else if (format === 'csv') { ext = 'csv'; - } else if (format = 'turtle') { + } else if (format === 'turtle') { + await removePrefixes(); ext = 'ttl'; } else if (format === 'ntriples') { ext = 'n3'; } - exec('cat raw_* > linkedConnections.' + ext, { cwd: output }, err => { - if (err) { - throw err; - } + try { + await exec(`cat raw_* > linkedConnections.${ext}`, { cwd: output }); let t1 = new Date(); console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms'); - del([ + await del([ path + '/connections_*', path + '/trips_*', output + '/.stops', @@ -120,11 +125,11 @@ Mapper.prototype.resultStream = function (path, output, done) { output + '/.services', output + '/raw_*' ], - { force: true } - ).then(function () { - done(output + '/linkedConnections.' + ext); - }); - }); + { force: true }); + done(`${output}/linkedConnections.${ext}`); + } catch (err) { + throw err; + } } }).on('error', err => { console.error(err); @@ -137,6 +142,25 @@ Mapper.prototype.resultStream = function (path, output, done) { } }; + var appendLineBreaks = async () => { + let files = (await readdir(output)).filter(raw => raw.startsWith('raw_')); + for (const [i, f] of files.entries()) { + if (i < files.length - 1) { + await appendFile(`${output}/${f}`, '\n') + } + } + }; + + var removePrefixes = async () => { + let files = (await readdir(output)).filter(raw => raw.startsWith('raw_')); + for (const [i, f] of files.entries()) { + if (i > 0) { + // TODO: find a not hard-coded way to remove prefixes + await exec(`sed -i 1,4d ${f}`, { cwd: output }); + } + } + }; + var servicesIterator = AsyncIterator.wrap(services); var routesIterator = AsyncIterator.wrap(routes); var stopsIterator = AsyncIterator.wrap(stops); @@ -277,15 +301,11 @@ if (!isMainThread) { connectionStream = connectionStream.pipe(new Connections2CSV(header)) .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.csv')); } else if (format === 'turtle') { - let prefixes = undefined; - - if (workerData['instance'] === 0) { - prefixes = { - lc: 'http://semweb.mmlab.be/ns/linkedconnections#', - gtfs: 'http://vocab.gtfs.org/terms#', - xsd: 'http://www.w3.org/2001/XMLSchema#' - }; - } + let prefixes = { + lc: 'http://semweb.mmlab.be/ns/linkedconnections#', + gtfs: 'http://vocab.gtfs.org/terms#', + xsd: 'http://www.w3.org/2001/XMLSchema#' + }; connectionStream = connectionStream.pipe(new Connections2Triples(workerData['options']['baseUris'], stopsdb)) .pipe(new N3.StreamWriter({ prefixes: prefixes })) .pipe(fs.createWriteStream(workerData['output'] + '/raw_' + workerData['instance'] + '.ttl')); diff --git a/package.json b/package.json index 1851fae..aa76557 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ "linkedconnections-joinandsort": "./bin/linkedconnections-sort.sh" }, "scripts": { - "test": "rm -rf test/sample-feed/ ; cp -r test/sample-feed-test/ test/sample-feed/ ; ./bin/gtfs2lc-sort.sh test/sample-feed ; ./bin/gtfs2lc.js -s -f jsonld test/sample-feed > test/sample-feed/connections-notjoined.nldjsonld ; ./bin/linkedconnections-sort.sh test/sample-feed/connections-notjoined.nldjsonld > test/sample-feed/connections.nldjsonld ; jest", - "coveralls": "jest --coverage && cat ./coverage/lcov.info | coveralls" + "test": "rm -rf test/sample-feed/ ; cp -r test/sample-feed-test/ test/sample-feed/ ; ./bin/gtfs2lc-sort.sh test/sample-feed ; ./bin/gtfs2lc.js -s -f jsonld test/sample-feed > test/sample-feed/connections-notjoined.nldjsonld ; ./bin/linkedconnections-sort.sh test/sample-feed/connections-notjoined.nldjsonld > test/sample-feed/connections.nldjsonld ; jest --runInBand", + "coveralls": "jest --runInBand --coverage && cat ./coverage/lcov.info | coveralls" }, "repository": { "type": "git", diff --git a/test/dataConversion.test.js b/test/dataConversion.test.js new file mode 100644 index 0000000..c1f83ef --- /dev/null +++ b/test/dataConversion.test.js @@ -0,0 +1,70 @@ +const c2csv = require('../lib/Connections2CSV'); +const fs = require('fs'); +const util = require('util'); +const { exec } = require('child_process'); +const { Readable } = require('stream'); + +const readFile = util.promisify(fs.readFile); + +beforeAll(async () => { + await doGtfsSort(); + await doBasicParsing(); +}); + +test('Convert connections to csv', async () => { + let csvCxs = await stream2Array(new c2csv()); + expect(csvCxs.length).toBeGreaterThan(0); + expect(csvCxs[0].split(',').length).toBe(7); +}); + +function doGtfsSort() { + return new Promise((resolve, reject) => { + exec(`./bin/gtfs2lc-sort.sh test/sample-feed`, (err, stdout, stderr) => { + if (err) { + reject(stderr); + } else { + resolve(); + } + }); + }); +} + +function doBasicParsing() { + return new Promise((resolve, reject) => { + exec(`./bin/gtfs2lc.js -s test/sample-feed > test/sample-feed/formats.json`, + async (err, stdout, stderr) => { + if (err) { + reject(stderr); + } else { + resolve(); + } + }); + }); +} + +async function* connGenerator() { + let conns = (await readFile('test/sample-feed/formats.json', 'utf8')).split('\n'); + for (const c of conns) { + try { + let jcx = JSON.parse(c); + jcx['departureTime'] = new Date(jcx['departureTime']); + jcx['arrivalTime'] = new Date(jcx['arrivalTime']); + yield jcx; + } catch (err) { + console.error(err); + } + } +} + +function stream2Array(stream) { + return new Promise((resolve, reject) => { + let array = []; + Readable.from(connGenerator()).pipe(stream) + .on('data', data => { + array.push(data); + }) + .on('end', () => { + resolve(array); + }); + }); +} diff --git a/test/resultStream.test.js b/test/resultStream.test.js index 3a6122e..78a6e0f 100644 --- a/test/resultStream.test.js +++ b/test/resultStream.test.js @@ -1,4 +1,3 @@ -const gtfs2lc = require('../lib/gtfs2lc.js'); const assert = require('assert'); const { exec } = require('child_process'); const fs = require('fs'); @@ -38,13 +37,9 @@ describe('Testing whether result contains certain objects (regression tests)', ( var connections; //This will be the first element when sorted correctly it('Stream should contain a first connection with arrivalStop AMV', async () => { - try { await doGtfsSort(); connections = await lcstreamToArray({}, 'result.json'); assert.equal(JSON.parse(connections[0])['arrivalStop'], 'AMV'); - } catch (err) { - console.error(err); - } }); it('JSON-LD Stream should contain Connections and use KeyvStore for data storage', async () => { From 24de1c7ac1972e959ade5b7f3f72ef90dc4a885c Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Sat, 15 Feb 2020 02:43:08 +0100 Subject: [PATCH 3/6] Increase test coverage --- test/URIStrategy.test.js | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/test/URIStrategy.test.js b/test/URIStrategy.test.js index 840d422..3ee7898 100644 --- a/test/URIStrategy.test.js +++ b/test/URIStrategy.test.js @@ -4,7 +4,8 @@ const URIStrategy = require('../lib/URIStrategy'); describe('URIStrategy', () => { describe('getRouteId', () => { it('should replace {routes.route_id} by connection.trip.route.route_id', () => { - const strategy = new URIStrategy({ + let strategy = new URIStrategy(); + strategy = new URIStrategy({ route: 'http://example.org/routes/{routes.route_id}', }); @@ -62,6 +63,33 @@ describe('URIStrategy', () => { 'http://example.org/routes/B1234' ); }); + + it('Should resolve stop URI', async () => { + let stops = new Map(); + stops.set('1', { stop_id: 'stop1' }); + const strategy = new URIStrategy({ + stop: 'http://example.org/stops/{stops.stop_id}' + }, stops); + assert.equal(await strategy.getStopId('1'), 'http://example.org/stops/stop1'); + // Should not resolve stop URI + assert.rejects( + async () => { await strategy.getStopId('2') }, + Error + ); + }); + + it('Should resolve trip URI', () => { + const strategy = new URIStrategy({ + trip: 'http://example.org/trips/{trips.trip_id}/{trips.startTime(yyyyMMdd)}', + }); + const connection = { + trip: { + trip_id: 'trip1', + startTime: new Date('2020-02-15T08:00:00.000Z') + } + }; + assert.equal(strategy.getTripId(connection), 'http://example.org/trips/trip1/20200215'); + }); }); describe('getId', () => { From fa5b250ed391e8cb2861735c45290afd8ce239e5 Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Sat, 15 Feb 2020 02:57:03 +0100 Subject: [PATCH 4/6] Increase coverage more --- test/URIStrategy.test.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/URIStrategy.test.js b/test/URIStrategy.test.js index 3ee7898..9fc1982 100644 --- a/test/URIStrategy.test.js +++ b/test/URIStrategy.test.js @@ -80,15 +80,17 @@ describe('URIStrategy', () => { it('Should resolve trip URI', () => { const strategy = new URIStrategy({ - trip: 'http://example.org/trips/{trips.trip_id}/{trips.startTime(yyyyMMdd)}', + trip: 'http://example.org/trips/{trips.trip_id}/{trips.startTime(yyyyMMdd)}/{connection.departureTime(HH)}{connection.arrivalTime(mm)}', }); const connection = { + departureTime: new Date('2020-02-15T09:23:00.000Z'), + arrivalTime: new Date('2020-02-15T09:42:00.000Z'), trip: { trip_id: 'trip1', startTime: new Date('2020-02-15T08:00:00.000Z') } }; - assert.equal(strategy.getTripId(connection), 'http://example.org/trips/trip1/20200215'); + assert.equal(strategy.getTripId(connection), 'http://example.org/trips/trip1/20200215/1042'); }); }); @@ -96,7 +98,7 @@ describe('URIStrategy', () => { it('should resolve expression using date-fns.format function', () => { const strategy = new URIStrategy({ connection: - 'http://example.org/connections/{trip_startTime}/{departureStop}/{trip_id}', + 'http://example.org/connections/{trip_startTime}/{departureStop}/{trip_id}{connection.something}', resolve: { trip_id: 'connection.trip.trip_id', trip_startTime: 'format(connection.trip.startTime, "yyyyMMdd\'T\'HHmm");', @@ -105,6 +107,7 @@ describe('URIStrategy', () => { }); const connection = { + something: 'some', departureStop: '1234', trip: { trip_id: '5678', @@ -114,7 +117,7 @@ describe('URIStrategy', () => { assert.equal( strategy.getId(connection), - 'http://example.org/connections/20180921T1025/1234/5678' + 'http://example.org/connections/20180921T1025/1234/5678some' ); }); }); From 6d58a86c4dcd4927587fff11014f4a964bb9a858 Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Sat, 15 Feb 2020 03:01:59 +0100 Subject: [PATCH 5/6] Correct tests --- test/URIStrategy.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/URIStrategy.test.js b/test/URIStrategy.test.js index 9fc1982..2ee404b 100644 --- a/test/URIStrategy.test.js +++ b/test/URIStrategy.test.js @@ -80,7 +80,7 @@ describe('URIStrategy', () => { it('Should resolve trip URI', () => { const strategy = new URIStrategy({ - trip: 'http://example.org/trips/{trips.trip_id}/{trips.startTime(yyyyMMdd)}/{connection.departureTime(HH)}{connection.arrivalTime(mm)}', + trip: 'http://example.org/trips/{trips.trip_id}/{trips.startTime(yyyyMMdd)}/{connection.departureTime(yy)}{connection.arrivalTime(yy)}', }); const connection = { departureTime: new Date('2020-02-15T09:23:00.000Z'), @@ -90,7 +90,7 @@ describe('URIStrategy', () => { startTime: new Date('2020-02-15T08:00:00.000Z') } }; - assert.equal(strategy.getTripId(connection), 'http://example.org/trips/trip1/20200215/1042'); + assert.equal(strategy.getTripId(connection), 'http://example.org/trips/trip1/20200215/2020'); }); }); From 5f9db7af6629e5108ff670d4b9bafacc6dfb141f Mon Sep 17 00:00:00 2001 From: Julian Rojas Date: Sat, 15 Feb 2020 03:08:12 +0100 Subject: [PATCH 6/6] Update dependencies and fix #97 and #63 --- package-lock.json | 253 ++++++++++++++++++++-------------------------- package.json | 10 +- 2 files changed, 115 insertions(+), 148 deletions(-) diff --git a/package-lock.json b/package-lock.json index 92211b2..21620d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -253,9 +253,9 @@ } }, "@fast-csv/format": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.0.2.tgz", - "integrity": "sha512-OqnJINuEdlmXTqEkSQXBCb2oR/4GcAV+Smy38q07Wdz2JSYw55O4McppteVcAYbp7eK+bgoha3ZGTC+R3GTUfg==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.0.3.tgz", + "integrity": "sha512-vJuqFUQDMnawfoLQMhwj1Of+VaHzHjHGWs4/kTG0AEyavfrFguMNmSvl7Ap07AWiyDO1HvAmuUZBKy8Nj0nVNA==", "requires": { "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", @@ -265,9 +265,9 @@ } }, "@fast-csv/parse": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.0.2.tgz", - "integrity": "sha512-QQ9Vqn/QWQ/IcxMz5TlhJcr89b0A0Tclo5P1JdkYrSZB4VvyOHspM/EHqNJ8p4smesLDxw0b4flMxsTITSBl0g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.1.0.tgz", + "integrity": "sha512-9iM0mhApo6qS1ZmZiNovPpYedHmZJHQOZYK2bEqkuAqoDD3YldfkK0hSFci41f7Wt0WzDWBpDJLeQJmoTd2M2g==", "requires": { "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", @@ -507,25 +507,25 @@ } }, "@nodelib/fs.scandir": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.1.tgz", - "integrity": "sha512-NT/skIZjgotDSiXs0WqYhgcuBKhUMgfekCmCGtkUAiLqZdOnrdjmZr9wRl3ll64J9NF79uZ4fk16Dx0yMc/Xbg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", "requires": { - "@nodelib/fs.stat": "2.0.1", + "@nodelib/fs.stat": "2.0.3", "run-parallel": "^1.1.9" } }, "@nodelib/fs.stat": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.1.tgz", - "integrity": "sha512-+RqhBlLn6YRBGOIoVYthsG0J9dfpO79eJyN7BYBkZJtfqrBwf2KK+rD/M/yjZR6WBmIhAgOV7S60eCgaSWtbFw==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==" }, "@nodelib/fs.walk": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.2.tgz", - "integrity": "sha512-J/DR3+W12uCzAJkw7niXDcqcKBg6+5G5Q/ZpThpGNzAUz70eOR6RV4XnnSN01qHZiVl0eavoxJsBypQoKsV2QQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", "requires": { - "@nodelib/fs.scandir": "2.1.1", + "@nodelib/fs.scandir": "2.1.3", "fastq": "^1.6.0" } }, @@ -631,9 +631,9 @@ "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" }, "@types/node": { - "version": "12.12.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.25.tgz", - "integrity": "sha512-nf1LMGZvgFX186geVZR1xMZKKblJiRfiASTHw85zED2kI1yDKHDwTKMdkaCbTlXoRKlGKaDfYywt+V0As30q3w==" + "version": "13.7.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.1.tgz", + "integrity": "sha512-Zq8gcQGmn4txQEJeiXo/KiLpon8TzAl0kmKH4zdWctPj05nWwp1ClMdAVEloqrQKfaC48PNLdgN/aVaLqUrluA==" }, "@types/stack-utils": { "version": "1.0.1", @@ -711,6 +711,15 @@ "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", "dev": true }, + "aggregate-error": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", + "integrity": "sha512-quoaXsZ9/BLNae5yiNoUz+Nhkwz83GhWwtYFglcjEQB2NDHCIpApbqXxIFnm4Pq/Nvhrsq5sYJFyohrrxnTGAA==", + "requires": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + } + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -981,7 +990,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "requires": { "fill-range": "^7.0.1" } @@ -1106,6 +1114,11 @@ } } }, + "clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==" + }, "cliui": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", @@ -1162,9 +1175,9 @@ } }, "commander": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.0.tgz", - "integrity": "sha512-NIQrwvv9V39FHgGFm36+U9SMQzbiHvU79k+iADraJTpmrFFfx7Ds0IvDoAdZsDrknlkRk14OYoWXb57uTh7/sw==" + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==" }, "component-emitter": { "version": "1.3.0", @@ -1356,15 +1369,25 @@ } }, "del": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-5.0.0.tgz", - "integrity": "sha512-TfU3nUY0WDIhN18eq+pgpbLY9AfL5RfiE9czKaTSolc6aK7qASXfDErvYgjV1UqCR4sNXDoxO0/idPmhDUt2Sg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/del/-/del-5.1.0.tgz", + "integrity": "sha512-wH9xOVHnczo9jN2IW68BabcecVPxacIA3g/7z6vhSU/4stOKQzeCRK0yD0A24WiAAUJmmVpWqrERcTxnLo3AnA==", "requires": { - "globby": "^10.0.0", - "is-path-cwd": "^2.0.0", - "is-path-in-cwd": "^2.0.0", - "p-map": "^2.0.0", - "rimraf": "^2.6.3" + "globby": "^10.0.1", + "graceful-fs": "^4.2.2", + "is-glob": "^4.0.1", + "is-path-cwd": "^2.2.0", + "is-path-inside": "^3.0.1", + "p-map": "^3.0.0", + "rimraf": "^3.0.0", + "slash": "^3.0.0" + }, + "dependencies": { + "graceful-fs": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" + } } }, "delayed-stream": { @@ -1390,13 +1413,6 @@ "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", "requires": { "path-type": "^4.0.0" - }, - "dependencies": { - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" - } } }, "domexception": { @@ -1711,13 +1727,20 @@ "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-csv": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.0.2.tgz", - "integrity": "sha512-ra9uIiXDEY1pRNnJJKPDtpL3ij9A1r4mCBef893yRkY4O4L0uWlpsZjEMPBME7Te9aqWNin6mbnqLG/QDpqw3A==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.1.0.tgz", + "integrity": "sha512-jTAMSDoQ39uPmAuFqE+LloG6Aep+4ylcjP6KI1XBSczJUc/hFv/IISxDFtXXTcECzTQh6RJWvqx+ZRxivggOUw==", "requires": { - "@fast-csv/format": "^4.0.2", - "@fast-csv/parse": "^4.0.2", + "@fast-csv/format": "^4.0.3", + "@fast-csv/parse": "^4.1.0", "@types/node": "^12.12.17" + }, + "dependencies": { + "@types/node": { + "version": "12.12.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.27.tgz", + "integrity": "sha512-odQFl/+B9idbdS0e8IxDl2ia/LP8KZLXhV3BUeI98TrZp0uoIzQPhGd+5EtzHmT0SMOIaPd7jfz6pOHLWTtl7A==" + } } }, "fast-deep-equal": { @@ -1726,56 +1749,15 @@ "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" }, "fast-glob": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.0.4.tgz", - "integrity": "sha512-wkIbV6qg37xTJwqSsdnIphL1e+LaGz4AIQqr00mIubMaEhv1/HEmJ0uuCGZRNRUkZZmOB5mJKO0ZUTVq+SxMQg==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.1.1.tgz", + "integrity": "sha512-nTCREpBY8w8r+boyFYAx21iL6faSsQynliPHM4Uf56SbkyohCNxpVPEH9xrF5TXKy+IsjkPUHDKiUkzBVRXn9g==", "requires": { - "@nodelib/fs.stat": "^2.0.1", - "@nodelib/fs.walk": "^1.2.1", - "glob-parent": "^5.0.0", - "is-glob": "^4.0.1", - "merge2": "^1.2.3", + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", "micromatch": "^4.0.2" - }, - "dependencies": { - "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "requires": { - "fill-range": "^7.0.1" - } - }, - "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "requires": { - "to-regex-range": "^5.0.1" - } - }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" - }, - "micromatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", - "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", - "requires": { - "braces": "^3.0.1", - "picomatch": "^2.0.5" - } - }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "requires": { - "is-number": "^7.0.0" - } - } } }, "fast-json-stable-stringify": { @@ -1810,7 +1792,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "requires": { "to-regex-range": "^5.0.1" } @@ -1942,9 +1923,9 @@ } }, "glob-parent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.0.0.tgz", - "integrity": "sha512-Z2RwiujPRGluePM6j699ktJYxmPpJKCfpGA13jz2hmFZC7gKetzrWvg5KN3+OsIFmydGyZ1AVwERCq1w/ZZwRg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.0.tgz", + "integrity": "sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw==", "requires": { "is-glob": "^4.0.1" } @@ -1956,9 +1937,9 @@ "dev": true }, "globby": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.1.tgz", - "integrity": "sha512-sSs4inE1FB2YQiymcmTv6NWENryABjUNPeWhOvmn4SjtKybglsyPZxFB3U1/+L1bYi0rNZDqCLlHyLYDl1Pq5A==", + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.2.tgz", + "integrity": "sha512-7dUi7RvCoT/xast/o/dLN53oqND4yk0nsHkhRgn9w65C4PofCLOoJ39iSOg+qVDdWQPIEj+eszMHQ+aLVwwQSg==", "requires": { "@types/glob": "^7.1.1", "array-union": "^2.1.0", @@ -1968,13 +1949,6 @@ "ignore": "^5.1.1", "merge2": "^1.2.3", "slash": "^3.0.0" - }, - "dependencies": { - "slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==" - } } }, "graceful-fs": { @@ -2117,9 +2091,9 @@ } }, "ignore": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.2.tgz", - "integrity": "sha512-vdqWBp7MyzdmHkkRWV5nY+PfGRbYbahfuvsBCh277tq+w9zyNi7h5CYJCK0kmzti9kU+O/cB7sE8HvKv6aXAKQ==" + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", + "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==" }, "immediate": { "version": "3.2.3", @@ -2142,6 +2116,11 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, + "indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2282,29 +2261,17 @@ "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "is-path-cwd": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==" }, - "is-path-in-cwd": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", - "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", - "requires": { - "is-path-inside": "^2.1.0" - } - }, "is-path-inside": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", - "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", - "requires": { - "path-is-inside": "^1.0.2" - } + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.2.tgz", + "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==" }, "is-plain-object": { "version": "2.0.4", @@ -3409,15 +3376,14 @@ "dev": true }, "merge2": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.2.3.tgz", - "integrity": "sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.3.0.tgz", + "integrity": "sha512-2j4DAdlBOkiSZIsaXk4mTE3sRS02yBHAtfy127xRV3bQUFqXkjHCHLW6Scv7DwNRbIWNHH8zpnz9zMaKXIdvYw==" }, "micromatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", - "dev": true, "requires": { "braces": "^3.0.1", "picomatch": "^2.0.5" @@ -3756,9 +3722,12 @@ } }, "p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "requires": { + "aggregate-error": "^3.0.0" + } }, "p-try": { "version": "2.2.0", @@ -3789,11 +3758,6 @@ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" - }, "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", @@ -3806,6 +3770,11 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", "dev": true }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -4070,9 +4039,9 @@ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" }, "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" } @@ -4329,8 +4298,7 @@ "slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==" }, "snapdragon": { "version": "0.8.2", @@ -4780,7 +4748,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "requires": { "is-number": "^7.0.0" } diff --git a/package.json b/package.json index aa76557..3306182 100644 --- a/package.json +++ b/package.json @@ -29,15 +29,15 @@ "dependencies": { "JSONStream": "^1.3.5", "asynciterator": "^2.0.1", - "commander": "^4.1.0", + "commander": "^4.1.1", "date-fns": "^2.9.0", - "fast-csv": "^4.0.2", + "del": "^5.1.0", + "fast-csv": "^4.1.0", "jsonld-stream": "^1.0.4", - "level": "^6.0.0", - "n3": "^1.3.5", - "del": "^5.0.0", "keyv": "^3.1.0", "keyv-file": "^0.1.13", + "level": "^6.0.0", + "n3": "^1.3.5", "uri-templates": "^0.2.0" }, "devDependencies": {