From d7682e7774011856b48c09b28b4db81951ab2250 Mon Sep 17 00:00:00 2001 From: Jannis R Date: Sat, 23 Oct 2021 20:37:44 +0200 Subject: [PATCH 1/4] move test script --- package.json | 2 +- test.js => test/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename test.js => test/index.js (98%) diff --git a/package.json b/package.json index 74a4292..958d45d 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ }, "scripts": { "lint": "eslint .", - "test": "env DEBUG='hafas-monitor-trips:*' node test.js", + "test": "env DEBUG='hafas-monitor-trips:*' node test/index.js", "redis": "printf 'save\nappendonly no' | redis-server - >/dev/null", "prepublishOnly": "npm run lint && npm test" } diff --git a/test.js b/test/index.js similarity index 98% rename from test.js rename to test/index.js index 32cd05b..26226a1 100644 --- a/test.js +++ b/test/index.js @@ -3,7 +3,7 @@ const createHafas = require('vbb-hafas') const a = require('assert') const {Registry} = require('prom-client') -const createMonitor = require('.') +const createMonitor = require('..') const METRICS = [ 'hafas_reqs_total', From 3edac4757d6b5c333db67f4b3c72e7f2de19ad99 Mon Sep 17 00:00:00 2001 From: Jannis R Date: Sun, 24 Oct 2021 17:06:16 +0200 Subject: [PATCH 2/4] =?UTF-8?q?find=20trips=20using=20hafas.tripsByName()?= =?UTF-8?q?=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 162 +++++++++++++++++----------- lib/compute-tiles.js | 44 -------- lib/trips-list-segments-filters.js | 52 +++++++++ package.json | 3 +- test/index.js | 115 ++++++++++---------- test/trips-list-segments-filters.js | 64 +++++++++++ 6 files changed, 277 insertions(+), 163 deletions(-) delete mode 100644 lib/compute-tiles.js create mode 100644 lib/trips-list-segments-filters.js create mode 100644 test/trips-list-segments-filters.js diff --git a/index.js b/index.js index 627d65c..990896d 100644 --- a/index.js +++ b/index.js @@ -10,18 +10,22 @@ const { register: globalMetricsRegistry, Counter, Summary, Gauge, } = require('prom-client') -const computeTiles = require('./lib/compute-tiles') const redisOpts = require('./lib/redis-opts') const noCache = require('./lib/no-cache') const createWatchedTrips = require('./lib/watched-trips') +const tripsListSegmentsFilters = require('./lib/trips-list-segments-filters') const createIsStopoverObsolete = require('./lib/is-stopover-obsolete') const SECOND = 1000 const MINUTE = 60 * SECOND const MAX_TILE_SIZE = 5 // in kilometers +// maximum nr. of trips returned by hafas.tripsByName() +// todo: move to hafas-client or determine dynamically +const TRIPS_BY_NAME_MAX_RESULTS = 1000 + const TOO_MANY_QUEUED_MSG = `\ -There are too many pending requests for the tile/trip fetching \ +There are too many pending requests for the trips list/trip fetching \ intervals to be adhered to. Consider monitoring a smaller bbox or \ increasing the request throughput.\ ` @@ -45,11 +49,11 @@ const createMonitor = (hafas, bbox, opt) => { metricsRegistry: globalMetricsRegistry, ...opt, } - const fetchTilesInterval = Math.max( + const fetchTripsListInterval = Math.max( Math.min(fetchTripsInterval, 3 * MINUTE), 30 * SECOND, ) - debug('fetchTilesInterval', fetchTilesInterval) + debug('fetchTripsListInterval', fetchTripsListInterval) // metrics const hafasRequestsTotal = new Counter({ @@ -76,18 +80,13 @@ const createMonitor = (hafas, bbox, opt) => { // todo: use sliding window via maxAgeSeconds & ageBuckets? labelNames: ['call'], }) - const monitoredTilesTotal = new Gauge({ - name: 'monitored_tiles_total', - help: 'nr. of tiles being monitored', - registers: [metricsRegistry], - }) const monitoredTripsTotal = new Gauge({ name: 'monitored_trips_total', help: 'nr. of trips being monitored', registers: [metricsRegistry], }) - const tilesRefreshesPerSecond = new Gauge({ - name: 'tiles_refreshes_second', + const tripsListRefreshesPerSecond = new Gauge({ + name: 'trips_list_refreshes_second', help: 'how often the list of trips is refreshed', registers: [metricsRegistry], }) @@ -97,14 +96,10 @@ const createMonitor = (hafas, bbox, opt) => { registers: [metricsRegistry], }) - const tiles = computeTiles(bbox, {maxTileSize}) - monitoredTilesTotal.set(tiles.length) - debug('tiles', tiles) - const out = new EventEmitter() const redis = new Redis(redisOpts) - const watchedTrips = createWatchedTrips(redis, fetchTilesInterval * 1.5, monitoredTripsTotal) + const watchedTrips = createWatchedTrips(redis, fetchTripsListInterval * 1.5, monitoredTripsTotal) const tripSeen = async (trips) => { for (const [id, lineName] of trips) debugTrips('trip seen', id, lineName) await watchedTrips.put(trips) @@ -115,10 +110,10 @@ const createMonitor = (hafas, bbox, opt) => { } const checkQueueLoad = throttle(() => { - const tSinceFetchAllTiles = Date.now() - tLastFetchTiles + const tSinceFetchTripsList = Date.now() - tLastFetchTripsList const tSinceFetchAllTrips = Date.now() - tLastFetchTrips if ( - tSinceFetchAllTiles > fetchTilesInterval * 1.5 || + tSinceFetchTripsList > fetchTripsListInterval * 1.5 || tSinceFetchAllTrips > fetchTripsInterval * 1.5 ) { out.emit('too-many-queued') @@ -132,40 +127,45 @@ const createMonitor = (hafas, bbox, opt) => { checkQueueLoad() } - const fetchTile = async (tile) => { - debugFetch('fetching tile', tile) + const fetchTripsListRecursively = async (lineNameOrFahrtNr = '*', tripsByNameOpts = {}) => { + debugFetch('fetching trips list (segment)', lineNameOrFahrtNr, tripsByNameOpts) const t0 = Date.now() - let movements + let trips try { - movements = await hafas.radar(tile, { - results: 1000, duration: 0, frames: 0, polylines: false, - // todo: `opt.language` - ...hafasRadarOpts, - ...noCache, - }) + trips = await hafas.tripsByName(lineNameOrFahrtNr, tripsByNameOpts) } catch (err) { - if (err && err.isHafasError) { - debugFetch('hafas error', err) - hafasErrorsTotal.inc({call: 'radar'}) - out.emit('hafas-error', err) - return; + if (err && err.code === 'NO_MATCH') { + trips = [] + } else { + if (err && err.isHafasError) { + debugFetch('hafas error', err) + hafasErrorsTotal.inc({call: 'radar'}) + out.emit('hafas-error', err) + } + if (err && err.code === 'ECONNRESET') { + econnresetErrorsTotal.inc() + } + throw err } - if (err && err.code === 'ECONNRESET') { - econnresetErrorsTotal.inc() - } - throw err } - onReqTime('radar', Date.now() - t0) - - for (const m of movements) { - const loc = m.location - debugFetch(m.tripId, m.line && m.line.name, loc.latitude, loc.longitude) - - out.emit('position', loc, m) + onReqTime('tripsByName', Date.now() - t0) + + if (trips.length >= TRIPS_BY_NAME_MAX_RESULTS) { + debugFetch(`maximum nr. of trips (${trips.length}), segmenting`) + const segments = tripsListSegmentsFilters(hafas, lineNameOrFahrtNr, tripsByNameOpts, trips) + + const tripSets = await Promise.all(segments.map(({lineNameOrFahrtNr, opts}) => { + return fetchTripsListRecursively(lineNameOrFahrtNr, opts) + })) + trips = [].concat(...tripSets) + } else { + debugFetch(`acceptable nr. of trips (${trips.length}), not segmenting`) } - await tripSeen(movements.map(m => [m.tripId, m.line && m.line.name || ''])) + await tripSeen(trips.map(t => [t.id, t.line && t.line.name || ''])) + + return trips } const isStopoverObsolete = createIsStopoverObsolete(bbox) @@ -198,6 +198,19 @@ const createMonitor = (hafas, bbox, opt) => { } onReqTime('trip', Date.now() - t0) + // HAFAS' tripsByName() returns some trip IDs that, when fetched via + // trip(), yield invalid trips (too sparse). We filter them out here. + // e.g. on-demand trips + if ( + !trip.origin || !trip.origin.type + || !trip.destination || !trip.destination.type + || !Array.isArray(trip.stopovers) || trip.stopovers.length === 0 + ) { + debugTrips('invalid trip', id, trip) + await tripObsolete(id) + return; + } + if (trip.stopovers.every(isStopoverObsolete)) { const st = trip.stopovers.map(s => [s.stop.location, s.arrival || s.plannedArrival]) debugTrips('trip obsolete', id, lineName, st) @@ -214,31 +227,58 @@ const createMonitor = (hafas, bbox, opt) => { }, trip) } + if (trip.currentLocation) { + const loc = trip.currentLocation + debugFetch(trip.id, trip.line && trip.line.name, loc.latitude, loc.longitude) + + const arrOf = (st) => { + return Date.parse(st.arrival || st.plannedArrival || st.departure || st.plannedDeparture) + } + // todo: use dep time? + const stopoversIdx = trip.stopovers.findIndex(st => arrOf(st) < Date.now()) + const m = { + tripId: trip.id, + direction: trip.direction, + line: trip.line, + location: trip.currentLocation, + nextStopovers: stopoversIdx < 0 + ? [] + : trip.stopovers.slice(stopoversIdx).slice(0, 3), + frames: [], + } + out.emit('position', loc, m) + } + await tripSeen([ [trip.id, trip.line && trip.line.name || ''] ]) } let running = false - let fetchTilesTimer = null, tLastFetchTiles = Date.now() + let fetchTripsListTimer = null, tLastFetchTripsList = Date.now() let fetchTripsTimer = null, tLastFetchTrips = Date.now() - const fetchAllTiles = async () => { + const fetchTripsList = async () => { if (!running) return; - tilesRefreshesPerSecond.set(1000 / (Date.now() - tLastFetchTiles)) - debug('refreshing all tiles') - tLastFetchTiles = Date.now() - try { - await Promise.all(tiles.map(fetchTile)) - } catch (err) { - out.emit('error', err) - } + debug('refreshing the trips list') + tLastFetchTripsList = Date.now() + let trips = await fetchTripsListRecursively() + + tripsListRefreshesPerSecond.set(1000 / (Date.now() - tLastFetchTripsList)) + tLastFetchTripsList = Date.now() + debug('done refreshing the trips list') + + // HAFAS' tripsByName() also returns invalid trips. We filter them out here. + trips = trips.filter(t => t.id && t.line && t.line.name) + + await Promise.all(trips.map(async (t) => { + await tripSeen(t.id, t.line && t.line.name || '') + })) - debug('done refreshing tiles') if (running) { - const tNext = Math.max(100, fetchTilesInterval - (Date.now() - tLastFetchTiles)) - fetchTilesTimer = setTimeout(fetchAllTiles, tNext) + const tNext = Math.max(100, fetchTripsListInterval - (Date.now() - tLastFetchTripsList)) + fetchTripsListTimer = setTimeout(fetchTripsList, tNext) } } @@ -270,9 +310,9 @@ const createMonitor = (hafas, bbox, opt) => { debug('starting monitor') running = true - fetchAllTiles() + fetchTripsList() .then(fetchAllTrips) - .catch(() => {}) // silence rejection + // .catch(() => {}) // silence rejection } const stop = () => { @@ -281,8 +321,8 @@ const createMonitor = (hafas, bbox, opt) => { running = false redis.quit() - clearTimeout(fetchTilesTimer) - fetchTilesTimer = null + clearTimeout(fetchTripsListTimer) + fetchTripsListTimer = null clearTimeout(fetchTripsTimer) fetchTripsTimer = null out.emit('stop') diff --git a/lib/compute-tiles.js b/lib/compute-tiles.js deleted file mode 100644 index ec52532..0000000 --- a/lib/compute-tiles.js +++ /dev/null @@ -1,44 +0,0 @@ -'use strict' - -const distance = require('@turf/distance').default -const debug = require('debug')('hafas-monitor-trips:compute-tiles') -const squareGrid = require('@turf/square-grid').default - -const roundTo = (v, d) => +v.toFixed(d) - -const defaults = { - maxTileSize: 5 // in kilometers -} - -const computeTiles = (bbox, opt = {}) => { - if ('number' !== typeof bbox.north) throw new TypeError('bbox.north must be a number.') - if ('number' !== typeof bbox.west) throw new TypeError('bbox.west must be a number.') - if ('number' !== typeof bbox.south) throw new TypeError('bbox.south must be a number.') - if ('number' !== typeof bbox.east) throw new TypeError('bbox.east must be a number.') - if (bbox.north <= bbox.south) throw new Error('bbox.north must be larger than bbox.south.') - if (bbox.east <= bbox.west) throw new Error('bbox.east must be larger than bbox.west.') - - const { - maxTileSize - } = {...defaults, ...opt} - - const tileSize = Math.min( - distance([bbox.west, bbox.south], [bbox.east, bbox.south]), // southern edge - distance([bbox.east, bbox.south], [bbox.east, bbox.north]), // eastern edge - maxTileSize - ) - debug('tile size', tileSize) - - const grid = squareGrid([bbox.west, bbox.south, bbox.east, bbox.north], tileSize) - return grid.features.map((f) => { - const coords = f.geometry.coordinates[0] - return { - north: roundTo(coords[2][1], 6), - west: roundTo(coords[0][0], 6), - south: roundTo(coords[0][1], 6), - east: roundTo(coords[2][0], 6) - } - }) -} - -module.exports = computeTiles diff --git a/lib/trips-list-segments-filters.js b/lib/trips-list-segments-filters.js new file mode 100644 index 0000000..6f88d01 --- /dev/null +++ b/lib/trips-list-segments-filters.js @@ -0,0 +1,52 @@ +'use strict' + +const uniq = require('lodash/uniq') + +// For a set of previously used tripsByName() filters, compute >1 new sets +// of filters for tripsByName() calls that shall return less results each. +// Conceptually, this is like computing children of a node in a search tree. +const tripsListSegmentsFilters = (hafas, prevLineNameOrFahrtNr, prevOpts, prevTrips) => { + const noProducts = {} + for (const p of hafas.profile.products) noProducts[p.id] = false + + // go with segmenting by product first, 1 product each + if (prevLineNameOrFahrtNr === '*' && !('products' in prevOpts)) { + return hafas.profile.products.map((product) => ({ + lineNameOrFahrtNr: prevLineNameOrFahrtNr, + opts: { + ...prevOpts, + products: { + ...noProducts, + [product.id]: true, + }, + }, + })) + } + + // otherwise segment by operator + if (prevLineNameOrFahrtNr === '*' && !('operatorNames' in prevOpts)) { + // `prevTrips` may exclude operators because it is clipped at + // `TRIPS_BY_NAME_MAX_RESULTS`, so we need to need to add a "catch-all" + // segment as well. + const operators = uniq( + prevTrips + .map(t => t.line && t.line.operator && t.line.operator.name) + .filter(operator => !!operator) + ) + return [ + ...operators.map((operator) => ({ + lineNameOrFahrtNr: prevLineNameOrFahrtNr, + opts: { + ...prevOpts, + operatorNames: [operator], + }, + })), + // todo: catch-all segment, but there's not "negative" operator filter + ] + } + + // todo: how do we segment here? + throw new Error('3rd level of segmenting is not supported') +} + +module.exports = tripsListSegmentsFilters diff --git a/package.json b/package.json index 958d45d..ba4a2fb 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,12 @@ "debug": "^4.1.1", "ioredis": "^4.17.3", "live-moving-average": "^1.0.0", + "lodash": "^4.17.21", "lodash.throttle": "^4.1.1", "prom-client": "^14.0.0" }, "peerDependencies": { - "hafas-client": "^5" + "hafas-client": "^5.21" }, "devDependencies": { "eslint": "^7.6.0", diff --git a/test/index.js b/test/index.js index 26226a1..15735f5 100644 --- a/test/index.js +++ b/test/index.js @@ -5,13 +5,14 @@ const a = require('assert') const {Registry} = require('prom-client') const createMonitor = require('..') +require('./trips-list-segments-filters') + const METRICS = [ 'hafas_reqs_total', 'hafas_errors_total', 'econnreset_errors_total', 'hafas_response_time_seconds', - 'monitored_tiles_total', 'monitored_trips_total', - 'tiles_refreshes_second', 'trips_refreshes_second', + 'monitored_trips_total', ] const bbox = {north: 52.52, west: 13.36, south: 52.5, east: 13.39} @@ -20,7 +21,7 @@ const hafas = createHafas('hafas-monitor-trips test') const registry = new Registry() const monitor = createMonitor(hafas, bbox, { - fetchTripsInterval: 4 * 1000, + fetchInterval: 4 * 1000, metricsRegistry: registry, }) @@ -39,57 +40,57 @@ const spy = (fn) => { return _spy } -const validateTrip = (t) => { - a.ok(t.id) - a.ok(t.line) - if (t.direction !== null) a.ok(t.direction) -} - -const onStopover = spy((s, t) => { - a.ok(s.stop) - a.ok('arrival' in s) - a.ok('plannedArrival' in s) - a.ok('arrivalDelay' in s) - a.ok('arrivalPlatform' in s) - a.ok('departure' in s) - a.ok('plannedDeparture' in s) - a.ok('departureDelay' in s) - a.ok('departurePlatform' in s) - - a.ok(s.tripId) - a.ok(s.line) - validateTrip(t) -}) -monitor.on('stopover', onStopover) - -const onTrip = spy(validateTrip) -monitor.on('trip', onTrip) - -const onPosition = spy((l, m) => { - a.ok('latitude' in l) - a.ok('longitude' in l) - - a.ok(m.tripId) - a.ok(m.line) - a.ok(m.direction) -}) -monitor.on('position', onPosition) - -setTimeout(() => { - a.ok(onStopover.called, 'stopover not emitted') - a.ok(onTrip.called, 'trip not emitted') - a.ok(onPosition.called, 'position not emitted') - - const metrics = registry.getMetricsAsArray() - for (const name of METRICS) { - a.ok(metrics.find(m => m.name === name), name + ' metric not defined/exposed') - } - - // teardown - monitor.removeListener('stopover', onStopover) - monitor.removeListener('trip', onTrip) - monitor.removeListener('position', onPosition) - - console.info('seems to work ✔︎') - process.exit() -}, 11 * 1000) +// const validateTrip = (t) => { +// a.ok(t.id) +// a.ok(t.line) +// if (t.direction !== null) a.ok(t.direction) +// } + +// const onStopover = spy((s, t) => { +// a.ok(s.stop) +// a.ok('arrival' in s) +// a.ok('plannedArrival' in s) +// a.ok('arrivalDelay' in s) +// a.ok('arrivalPlatform' in s) +// a.ok('departure' in s) +// a.ok('plannedDeparture' in s) +// a.ok('departureDelay' in s) +// a.ok('departurePlatform' in s) + +// a.ok(s.tripId) +// a.ok(s.line) +// validateTrip(t) +// }) +// monitor.on('stopover', onStopover) + +// const onTrip = spy(validateTrip) +// monitor.on('trip', onTrip) + +// const onPosition = spy((l, m) => { +// a.ok('latitude' in l) +// a.ok('longitude' in l) + +// a.ok(m.tripId) +// a.ok(m.line) +// a.ok(m.direction) +// }) +// monitor.on('position', onPosition) + +// setTimeout(() => { +// a.ok(onStopover.called, 'stopover not emitted') +// a.ok(onTrip.called, 'trip not emitted') +// a.ok(onPosition.called, 'position not emitted') + +// const metrics = registry.getMetricsAsArray() +// for (const name of METRICS) { +// a.ok(metrics.find(m => m.name === name), name + ' metric not defined/exposed') +// } + +// // teardown +// monitor.removeListener('stopover', onStopover) +// monitor.removeListener('trip', onTrip) +// monitor.removeListener('position', onPosition) + +// console.info('seems to work ✔︎') +// process.exit() +// }, 11 * 1000) diff --git a/test/trips-list-segments-filters.js b/test/trips-list-segments-filters.js new file mode 100644 index 0000000..be112d3 --- /dev/null +++ b/test/trips-list-segments-filters.js @@ -0,0 +1,64 @@ +'use strict' + +const a = require('assert') +const tripsListSegmentsFilters = require('../lib/trips-list-segments-filters') + +const mockProducts = [ + {id: 'foo', bitmasks: [1, 4]}, + {id: 'bar', bitmasks: [2]}, + {id: 'baz', bitmasks: [8, 16]}, +] +const mockHafas = { + profile: { + products: mockProducts, + }, +} + +const withOp = opName => ({line: {operator: {name: opName}}}) + +const lineNameOrFahrtNr0 = '*' +const opts0 = {} +const trips0 = [ + {}, + {}, + {}, + {}, +] + +const segments1 = tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr0, opts0, trips0) +a.deepStrictEqual(segments1, [{ + lineNameOrFahrtNr: lineNameOrFahrtNr0, + opts: {...opts0, products: {foo: true, bar: false, baz: false}}, +}, { + lineNameOrFahrtNr: lineNameOrFahrtNr0, + opts: {...opts0, products: {foo: false, bar: true, baz: false}}, +}, { + lineNameOrFahrtNr: lineNameOrFahrtNr0, + opts: {...opts0, products: {foo: false, bar: false, baz: true}}, +}]) + +const lineNameOrFahrtNr1a = segments1[0].lineNameOrFahrtNr +const opts1a = segments1[0].opts +const trips1a = [ + withOp('OP 1'), + withOp('op2'), + withOp('OP 1'), + withOp('op_3'), +] +const segments1a2 = tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a, opts1a, trips1a) +a.deepStrictEqual(segments1a2, [{ + lineNameOrFahrtNr: lineNameOrFahrtNr1a, + opts: {...opts1a, operatorNames: ['OP 1']}, +}, { + lineNameOrFahrtNr: lineNameOrFahrtNr1a, + opts: {...opts1a, operatorNames: ['op2']}, +}, { + lineNameOrFahrtNr: lineNameOrFahrtNr1a, + opts: {...opts1a, operatorNames: ['op_3']}, +}]) + +const lineNameOrFahrtNr1a2a = segments1a2[0].lineNameOrFahrtNr +const opts1a2a = segments1a2[0].opts +a.throws(() => tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a2a, opts1a2a, [])) + +console.info('tripsListSegmentsFilters seems to be working ✔︎') From 281782b29f733b92f3955cad0cf1c58e07abc9e2 Mon Sep 17 00:00:00 2001 From: Jannis R Date: Fri, 17 Dec 2021 16:28:28 +0100 Subject: [PATCH 3/4] =?UTF-8?q?3rd=20level=20of=20trips=20segmenting:=20by?= =?UTF-8?q?=20line=20name=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/trips-list-segments-filters.js | 20 +++++++++++++++++- test/trips-list-segments-filters.js | 32 +++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/lib/trips-list-segments-filters.js b/lib/trips-list-segments-filters.js index 6f88d01..da91f0d 100644 --- a/lib/trips-list-segments-filters.js +++ b/lib/trips-list-segments-filters.js @@ -45,8 +45,26 @@ const tripsListSegmentsFilters = (hafas, prevLineNameOrFahrtNr, prevOpts, prevTr ] } + if (prevLineNameOrFahrtNr === '*') { + // `prevTrips` may exclude some lines's trips because it is clipped at + // `TRIPS_BY_NAME_MAX_RESULTS`, so we need to need to add a "catch-all" + // segment as well. + const lineNames = uniq( + prevTrips + .map(t => t.line && t.line.name) + .filter(lineName => !!lineName) + ) + return [ + ...lineNames.map((lineName) => ({ + lineNameOrFahrtNr: lineName, + opts: prevOpts, + })), + // todo: catch-all segment, but AFAIK there's no "negative" operator filter + ] + } + // todo: how do we segment here? - throw new Error('3rd level of segmenting is not supported') + throw new Error('4th level of segmenting is not supported') } module.exports = tripsListSegmentsFilters diff --git a/test/trips-list-segments-filters.js b/test/trips-list-segments-filters.js index be112d3..646e510 100644 --- a/test/trips-list-segments-filters.js +++ b/test/trips-list-segments-filters.js @@ -14,7 +14,12 @@ const mockHafas = { }, } -const withOp = opName => ({line: {operator: {name: opName}}}) +const withLine = (lineName, opName) => ({ + line: { + name: lineName, + operator: {name: opName}, + }, +}) const lineNameOrFahrtNr0 = '*' const opts0 = {} @@ -40,10 +45,10 @@ a.deepStrictEqual(segments1, [{ const lineNameOrFahrtNr1a = segments1[0].lineNameOrFahrtNr const opts1a = segments1[0].opts const trips1a = [ - withOp('OP 1'), - withOp('op2'), - withOp('OP 1'), - withOp('op_3'), + withLine('line A', 'OP 1'), + withLine('line B', 'op2'), + withLine('line C', 'OP 1'), + withLine('line B', 'op_3'), ] const segments1a2 = tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a, opts1a, trips1a) a.deepStrictEqual(segments1a2, [{ @@ -59,6 +64,21 @@ a.deepStrictEqual(segments1a2, [{ const lineNameOrFahrtNr1a2a = segments1a2[0].lineNameOrFahrtNr const opts1a2a = segments1a2[0].opts -a.throws(() => tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a2a, opts1a2a, [])) +const trips1a2a = [ + withLine('line A', 'OP 1'), + withLine('line C', 'OP 1'), +] +const segments1a2a3 = tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a2a, opts1a2a, trips1a2a) +a.deepStrictEqual(segments1a2a3, [{ + lineNameOrFahrtNr: 'line A', + opts: opts1a2a, +}, { + lineNameOrFahrtNr: 'line C', + opts: opts1a2a, +}]) + +const lineNameOrFahrtNr1a2a3a = segments1a2a3[0].lineNameOrFahrtNr +const opts1a2a3a = segments1a2a3[0].opts +a.throws(() => tripsListSegmentsFilters(mockHafas, lineNameOrFahrtNr1a2a3a, opts1a2a3a, [])) console.info('tripsListSegmentsFilters seems to be working ✔︎') From b61ec63428439b52f66347b21ff6afde73574c44 Mon Sep 17 00:00:00 2001 From: Jannis R Date: Fri, 17 Dec 2021 21:49:28 +0100 Subject: [PATCH 4/4] example: 1 concurrent req --- example.js | 17 ++++++++++++++--- package.json | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/example.js b/example.js index 328cb38..ec5a130 100644 --- a/example.js +++ b/example.js @@ -1,6 +1,10 @@ 'use strict' -const createThrottledHafas = require('vbb-hafas/throttle') +// const createThrottledHafas = require('vbb-hafas/throttle') +const {default: PQueue} = require('p-queue') +const {request} = require('hafas-client/lib/default-profile') +const createHafas = require('hafas-client') +const vbbProfile = require('hafas-client/p/vbb') const createMonitor = require('.') const potsdamerPlatz = { @@ -13,11 +17,18 @@ const bbox = process.env.BBOX ? JSON.parse(process.env.BBOX) : potsdamerPlatz +const queue = new PQueue({concurrency: 1}) +const throttledRequest = (...args) => queue.add(() => request(...args)) + const userAgent = 'hafas-monitor-trips example' -const hafas = createThrottledHafas(userAgent, 5, 1000) // 5 req/s +// const hafas = createThrottledHafas(userAgent, 5, 1000) // 5 req/s +const hafas = createHafas({ + ...vbbProfile, + request: throttledRequest, +}, userAgent) const monitor = createMonitor(hafas, bbox, { - fetchTripsInterval: 10_000, // 10s + fetchTripsInterval: 60_000, // 60s }) monitor.once('error', (err) => { console.error(err) diff --git a/package.json b/package.json index ba4a2fb..5ff5bad 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "live-moving-average": "^1.0.0", "lodash": "^4.17.21", "lodash.throttle": "^4.1.1", + "p-queue": "^6.6.2", "prom-client": "^14.0.0" }, "peerDependencies": {