From 8838101b973e1fa4f4508aca0c9ea1c3001ea11d Mon Sep 17 00:00:00 2001 From: Robert Stoll Date: Wed, 22 Dec 2021 17:28:36 +0100 Subject: [PATCH] #148 fetch processing errors by bound --- server/api/controllers/controller.ts | 41 +++++++------------ .../processing-errors.controller.ts | 3 +- .../services/generateLocationData.service.ts | 14 +++++-- server/api/services/locationCache.ts | 8 ++-- server/common/build.info.ts | 6 +-- server/common/swagger/Api.yaml | 14 +++++-- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/server/api/controllers/controller.ts b/server/api/controllers/controller.ts index 530a04ee..9503d1ba 100644 --- a/server/api/controllers/controller.ts +++ b/server/api/controllers/controller.ts @@ -17,8 +17,10 @@ import { getFountainFromCacheIfNotForceRefreshOrFetch, getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate, populateCacheWithCities as populateLocationCacheWithCities, + getProcessingErrorsByBoundingBox, } from '../services/generateLocationData.service'; import { illegalState } from '../../common/illegalState'; +import { tileToLocationCacheKey } from '../services/locationCache'; export class Controller { constructor() { @@ -60,9 +62,7 @@ export class Controller { async getByBounds(req: Request, res: Response, next: ErrorHandler): Promise { handlingErrors(next, () => { - const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw')); - const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne')); - const boundingBox = BoundingBox(southWest, northEast); + const boundingBox = this.getBoundingBoxFromQueryParam(req); const essential = getSingleBooleanQueryParam(req, 'essential'); const refresh = getSingleBooleanQueryParam(req, 'refresh'); @@ -70,6 +70,13 @@ export class Controller { }); } + private getBoundingBoxFromQueryParam(req: Request): BoundingBox { + const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw')); + const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne')); + const boundingBox = BoundingBox(southWest, northEast); + return boundingBox; + } + private async byBoundingBox( res: Response, boundingBox: BoundingBox, @@ -112,29 +119,11 @@ export class Controller { /** * Function to extract processing errors from detailed list of fountains */ - async getProcessingErrors(_req: Request, res: Response): Promise { - //TODO #148 return processing errors for given boundingBox - sendJson(res, [], 'not yet implemented'); - // returns all processing errors for a given location - // made for #206 - // const city = getSingleStringQueryParam(req, 'city'); - // const key = city + '_errors'; - - // if (locationCache.keys().indexOf(key) < 0) { - // // if data not in cache, create error list - // locationCache.set(key, extractProcessingErrors(locationCache.get(city))); - // } - // locationCache.get(key, (err, value) => { - // if (!err) { - // sendJson(res, value, 'cityCache.get ' + key); - // l.info('controller.js: getProcessingErrors !err sent'); - // } else { - // const errMsg = 'Error with cache: ' + err; - // l.info('controller.js: getProcessingErrors ' + errMsg); - // res.statusMessage = errMsg; - // res.status(500).send(err.stack); - // } - // }); + getProcessingErrors(req: Request, res: Response): void { + // returns all processing errors for a given location made for #206 + const boundingBox = this.getBoundingBoxFromQueryParam(req); + const errors = getProcessingErrorsByBoundingBox(boundingBox); + sendJson(res, errors ?? [], tileToLocationCacheKey(boundingBox)); } } export const controller = new Controller(); diff --git a/server/api/controllers/processing-errors.controller.ts b/server/api/controllers/processing-errors.controller.ts index e945fe52..8f45afd9 100644 --- a/server/api/controllers/processing-errors.controller.ts +++ b/server/api/controllers/processing-errors.controller.ts @@ -10,12 +10,13 @@ import _ from 'lodash'; import { l } from '../../common/logger'; import { FountainCollection } from '../../common/typealias'; +import '../../common/importAllExtensions'; //TODO @ralfhauser, I don't know the type of errorCollection since I was not able to get a real example of an issue, please narrow down accordingly export type ProcessingError = any; export function hasProcessingIssues(obj: Record): obj is { issues: ProcessingError[] } { - return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues); + return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues) && obj.issues.nonEmpty(); } export function extractProcessingErrors(fountainCollection: FountainCollection | undefined): ProcessingError[] { diff --git a/server/api/services/generateLocationData.service.ts b/server/api/services/generateLocationData.service.ts index cb732f02..c48463c5 100644 --- a/server/api/services/generateLocationData.service.ts +++ b/server/api/services/generateLocationData.service.ts @@ -20,7 +20,7 @@ import { import { BoundingBox, Database, Fountain, FountainCollection, LngLat } from '../../common/typealias'; import { MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; import sharedConstants from '../../common/shared-constants'; -import { extractProcessingErrors } from '../controllers/processing-errors.controller'; +import { extractProcessingErrors, ProcessingError } from '../controllers/processing-errors.controller'; import { illegalState } from '../../common/illegalState'; import '../../common/importAllExtensions'; import { @@ -30,6 +30,7 @@ import { getBoundingBoxOfTiles, getCachedEssentialFountainCollection, getCachedFullFountainCollection, + getCachedProcessingErrors, getTileOfLocation, locationCacheKeyToTile, splitInTiles, @@ -100,8 +101,14 @@ export async function getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( '\nend: ' + end.toISOString() ); + //TODO @ralf.hauser, derive lastScan from the cached collections (what date to use?) otherwise it will result in a + // different e-tag and client needs to refetch data even though it is still the same return FountainCollection(allFountains, /* lastScan= */ start); } +export function getProcessingErrorsByBoundingBox(boundingBox: BoundingBox): ProcessingError[] { + const arr = splitInTiles(boundingBox).map(tile => getCachedProcessingErrors(tile)); + return arr.reduce((acc, v) => (v !== undefined ? acc.concat(v.value) : acc), /*initial=*/ []); +} async function byTilesFromCacheIfNotForceRefreshOrPopulate( forceRefresh: boolean, @@ -155,8 +162,9 @@ async function fetchFountainsFromServerAndUpdateCache( ) ); - const collections = Array.from(groupedByTile.entries()).map(([cacheKey, fountains]) => { - const tile = locationCacheKeyToTile(cacheKey); + const collections = tiles.map(tile => { + const cacheKey = tileToLocationCacheKey(tile); + const fountains = groupedByTile.get(cacheKey) ?? []; let fountainCollection: FountainCollection | undefined = FountainCollection(fountains); updateCacheWithFountains(tile, fountainCollection, ttlInHours); fountainCollection = ( diff --git a/server/api/services/locationCache.ts b/server/api/services/locationCache.ts index aa37ea7a..11ed083c 100644 --- a/server/api/services/locationCache.ts +++ b/server/api/services/locationCache.ts @@ -30,7 +30,7 @@ function getCachedFountainCollection(tile: Tile, suffix: string): CacheEntry | undefined { - return locationCache.get>(tileToLocationCacheKey(tile)); + return locationCache.get>(tileToLocationCacheKey(tile) + PROCESSING_ERRORS_SUFFIX); } export function cacheFullFountainCollection(tile: Tile, fountainCollection: FountainCollection, ttl: number): void { @@ -100,9 +100,9 @@ export function tileToLocationCacheKey(tile: Tile): string { // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap // if you change something here, then you need to change it in proximap as well -// 0.01 lat is ~1km -const TILE_SIZE = 0.01; -const ROUND_FACTOR = 100; +// 0.01 lat is ~5km +const TILE_SIZE = 0.05; +const ROUND_FACTOR = 20; // 1/0.05; export const LNG_LAT_STRING_PRECISON = 2; function roundToTilePrecision(n: number): number { return Math.floor(n * ROUND_FACTOR) / ROUND_FACTOR; diff --git a/server/common/build.info.ts b/server/common/build.info.ts index 299f0242..c94fb06f 100644 --- a/server/common/build.info.ts +++ b/server/common/build.info.ts @@ -1,9 +1,9 @@ // this file is automatically generated by git.version.js script const buildInfo = { version: '', - revision: '11e097b', + revision: 'aa6848b', branch: '#148-load-by-bound', - commit_time: '2021-11-01 09:01:34 +0000', - build_time: 'Wed Dec 22 2021 16:21:24 GMT+0100 (Central European Standard Time)', + commit_time: '2021-12-22 16:49:55 +0100', + build_time: 'Wed Dec 22 2021 17:13:05 GMT+0100 (Central European Standard Time)', }; export default buildInfo; diff --git a/server/common/swagger/Api.yaml b/server/common/swagger/Api.yaml index aa5bcc4d..2989ec3a 100644 --- a/server/common/swagger/Api.yaml +++ b/server/common/swagger/Api.yaml @@ -126,12 +126,20 @@ paths: get: description: Fetch list of processing errors for given location parameters: - - name: city + - name: sw in: query type: string - example: ch-zh + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.3229261255644,8.45960259979614 required: true - description: name of location for which fountain processing errors are to be served + description: lat,lng of the south west location of the bounding box + - name: ne + in: query + type: string + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.431119712250506,8.61940272745742 + required: true + description: lat,lng of the north east location of the bounding box responses: 200: description: Returns a collection of processing errors.