water-fountains#148 fetch processing errors by bound

server/api/controllers/controller.ts

@@ -17,8 +17,10 @@ import {
   getFountainFromCacheIfNotForceRefreshOrFetch,
   getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate,
   populateCacheWithCities as populateLocationCacheWithCities,
+  getProcessingErrorsByBoundingBox,
 } from '../services/generateLocationData.service';
 import { illegalState } from '../../common/illegalState';
+import { tileToLocationCacheKey } from '../services/locationCache';
 
 export class Controller {
   constructor() {
@@ -60,16 +62,21 @@ export class Controller {
 
   async getByBounds(req: Request, res: Response, next: ErrorHandler): Promise<void> {
     handlingErrors(next, () => {
-      const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw'));
-      const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne'));
-      const boundingBox = BoundingBox(southWest, northEast);
+      const boundingBox = this.getBoundingBoxFromQueryParam(req);
       const essential = getSingleBooleanQueryParam(req, 'essential');
       const refresh = getSingleBooleanQueryParam(req, 'refresh');
 
       return this.byBoundingBox(res, boundingBox, essential, refresh);
     });
   }
 
+  private getBoundingBoxFromQueryParam(req: Request): BoundingBox {
+    const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw'));
+    const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne'));
+    const boundingBox = BoundingBox(southWest, northEast);
+    return boundingBox;
+  }
+
   private async byBoundingBox(
     res: Response,
     boundingBox: BoundingBox,
@@ -112,29 +119,11 @@ export class Controller {
   /**
    * Function to extract processing errors from detailed list of fountains
    */
-  async getProcessingErrors(_req: Request, res: Response): Promise<void> {
-    //TODO #148 return processing errors for given boundingBox
-    sendJson(res, [], 'not yet implemented');
-    // returns all processing errors for a given location
-    // made for #206
-    // const city = getSingleStringQueryParam(req, 'city');
-    // const key = city + '_errors';
-
-    // if (locationCache.keys().indexOf(key) < 0) {
-    //   // if data not in cache, create error list
-    //   locationCache.set<any[]>(key, extractProcessingErrors(locationCache.get<FountainCollection>(city)));
-    // }
-    // locationCache.get(key, (err, value) => {
-    //   if (!err) {
-    //     sendJson(res, value, 'cityCache.get ' + key);
-    //     l.info('controller.js: getProcessingErrors !err sent');
-    //   } else {
-    //     const errMsg = 'Error with cache: ' + err;
-    //     l.info('controller.js: getProcessingErrors ' + errMsg);
-    //     res.statusMessage = errMsg;
-    //     res.status(500).send(err.stack);
-    //   }
-    // });
+  getProcessingErrors(req: Request, res: Response): void {
+    // returns all processing errors for a given location made for #206
+    const boundingBox = this.getBoundingBoxFromQueryParam(req);
+    const errors = getProcessingErrorsByBoundingBox(boundingBox);
+    sendJson(res, errors ?? [], tileToLocationCacheKey(boundingBox));
   }
 }
 export const controller = new Controller();

server/api/controllers/processing-errors.controller.ts

@@ -10,12 +10,13 @@
 import _ from 'lodash';
 import { l } from '../../common/logger';
 import { FountainCollection } from '../../common/typealias';
+import '../../common/importAllExtensions';
 
 //TODO @ralfhauser, I don't know the type of errorCollection since I was not able to get a real example of an issue, please narrow down accordingly
 export type ProcessingError = any;
 
 export function hasProcessingIssues(obj: Record<string, unknown>): obj is { issues: ProcessingError[] } {
-  return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues);
+  return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues) && obj.issues.nonEmpty();
 }
 
 export function extractProcessingErrors(fountainCollection: FountainCollection | undefined): ProcessingError[] {

server/api/services/generateLocationData.service.ts

@@ -20,7 +20,7 @@ import {
 import { BoundingBox, Database, Fountain, FountainCollection, LngLat } from '../../common/typealias';
 import { MediaWikiSimplifiedEntity } from '../../common/wikimedia-types';
 import sharedConstants from '../../common/shared-constants';
-import { extractProcessingErrors } from '../controllers/processing-errors.controller';
+import { extractProcessingErrors, ProcessingError } from '../controllers/processing-errors.controller';
 import { illegalState } from '../../common/illegalState';
 import '../../common/importAllExtensions';
 import {
@@ -30,6 +30,7 @@ import {
   getBoundingBoxOfTiles,
   getCachedEssentialFountainCollection,
   getCachedFullFountainCollection,
+  getCachedProcessingErrors,
   getTileOfLocation,
   locationCacheKeyToTile,
   splitInTiles,
@@ -100,8 +101,14 @@ export async function getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate(
       '\nend:   ' +
       end.toISOString()
   );
+  //TODO @ralf.hauser, derive lastScan from the cached collections (what date to use?) otherwise it will result in a
+  // different e-tag and client needs to refetch data even though it is still the same
   return FountainCollection(allFountains, /* lastScan= */ start);
 }
+export function getProcessingErrorsByBoundingBox(boundingBox: BoundingBox): ProcessingError[] {
+  const arr = splitInTiles(boundingBox).map(tile => getCachedProcessingErrors(tile));
+  return arr.reduce((acc, v) => (v !== undefined ? acc.concat(v.value) : acc), /*initial=*/ []);
+}
 
 async function byTilesFromCacheIfNotForceRefreshOrPopulate(
   forceRefresh: boolean,
@@ -155,8 +162,9 @@ async function fetchFountainsFromServerAndUpdateCache(
     )
   );
 
-  const collections = Array.from(groupedByTile.entries()).map(([cacheKey, fountains]) => {
-    const tile = locationCacheKeyToTile(cacheKey);
+  const collections = tiles.map(tile => {
+    const cacheKey = tileToLocationCacheKey(tile);
+    const fountains = groupedByTile.get(cacheKey) ?? [];
     let fountainCollection: FountainCollection | undefined = FountainCollection(fountains);
     updateCacheWithFountains(tile, fountainCollection, ttlInHours);
     fountainCollection = (

server/api/services/locationCache.ts

@@ -30,7 +30,7 @@ function getCachedFountainCollection(tile: Tile, suffix: string): CacheEntry<Fou
 }
 
 export function getCachedProcessingErrors(tile: Tile): CacheEntry<ProcessingError> | undefined {
-  return locationCache.get<CacheEntry<FountainCollection>>(tileToLocationCacheKey(tile));
+  return locationCache.get<CacheEntry<ProcessingError>>(tileToLocationCacheKey(tile) + PROCESSING_ERRORS_SUFFIX);
 }
 
 export function cacheFullFountainCollection(tile: Tile, fountainCollection: FountainCollection, ttl: number): void {
@@ -100,9 +100,9 @@ export function tileToLocationCacheKey(tile: Tile): string {
 
 // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap
 // if you change something here, then you need to change it in proximap as well
-// 0.01 lat is ~1km
-const TILE_SIZE = 0.01;
-const ROUND_FACTOR = 100;
+// 0.01 lat is ~5km
+const TILE_SIZE = 0.05;
+const ROUND_FACTOR = 20; // 1/0.05;
 export const LNG_LAT_STRING_PRECISON = 2;
 function roundToTilePrecision(n: number): number {
   return Math.floor(n * ROUND_FACTOR) / ROUND_FACTOR;

server/common/build.info.ts

@@ -1,9 +1,9 @@
 // this file is automatically generated by git.version.js script
 const buildInfo = {
   version: '',
-  revision: '11e097b',
+  revision: 'aa6848b',
   branch: '#148-load-by-bound',
-  commit_time: '2021-11-01 09:01:34 +0000',
-  build_time: 'Wed Dec 22 2021 16:21:24 GMT+0100 (Central European Standard Time)',
+  commit_time: '2021-12-22 16:49:55 +0100',
+  build_time: 'Wed Dec 22 2021 17:13:05 GMT+0100 (Central European Standard Time)',
 };
 export default buildInfo;

server/common/swagger/Api.yaml

@@ -126,12 +126,20 @@ paths:
     get:
       description: Fetch list of processing errors for given location
       parameters:
-        - name: city
+        - name: sw
           in: query
           type: string
-          example: ch-zh
+          pattern: \d+(.\d+)?,\d+(\.d+)?
+          example: 47.3229261255644,8.45960259979614
           required: true
-          description: name of location for which fountain processing errors are to be served
+          description: lat,lng of the south west location of the bounding box
+        - name: ne
+          in: query
+          type: string
+          pattern: \d+(.\d+)?,\d+(\.d+)?
+          example: 47.431119712250506,8.61940272745742
+          required: true
+          description: lat,lng of the north east location of the bounding box
       responses:
         200:
           description: Returns a collection of processing errors.