diff --git a/.envTEMPLATE b/.envTEMPLATE index 73ff0ef3..b0ada1c8 100644 --- a/.envTEMPLATE +++ b/.envTEMPLATE @@ -2,7 +2,7 @@ APP_ID=datablue PORT=3000 LOG_LEVEL=debug -REQUEST_LIMIT=100kb +REQUEST_LIMIT=10MB GOOGLE_API_KEY=mykey SESSION_SECRET=mySecret NODE_ENV=development diff --git a/config/locations.ts b/config/locations.ts index 58cebfdd..4382ee49 100644 --- a/config/locations.ts +++ b/config/locations.ts @@ -1,4 +1,4 @@ -import { Translated } from '../server/common/typealias'; +import { BoundingBox, Translated, uncheckedBoundingBoxToChecked } from '../server/common/typealias'; /* * @license @@ -19,14 +19,16 @@ export interface Location { name: string; description: Translated; description_more: Translated; - bounding_box: BoundingBox; + bounding_box: UncheckedBoundingBox; + //TODO @ralf.hauser not used as it seems, remove? operator_fountain_catalog_qid: string; + //TODO @ralf.hauser not used as it seems, remove? issue_api: IssueApi; } // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap // if you change something here, then you need to change it in proximap as well -export interface BoundingBox { +export interface UncheckedBoundingBox { latMin: number; lngMin: number; latMax: number; @@ -521,7 +523,23 @@ export function isCity(s: string): s is City { return cities.includes(s as City); } +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export function getCityBoundingBox(city: City): BoundingBox { + const uncheckedBoundingBox = locationsCollection[city].bounding_box; + try { + return uncheckedBoundingBoxToChecked(uncheckedBoundingBox); + } catch (e: any) { + const newErr = new Error('Could not get city bounding box for ' + city); + newErr.stack += '\nCaused by: ' + e.stack; + throw newErr; + } +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well export type LocationsCollection = Record; + // we don't expose just the internal structure as we also want to be sure that it follows the spec. // However, we allow City union to grow dynamically export const locationsCollection: LocationsCollection = internalLocationsCollection; diff --git a/package-lock.json b/package-lock.json index 1a623c0f..afd82c82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2627,9 +2627,9 @@ } }, "@types/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", "dev": true, "requires": { "@types/connect": "*", @@ -2637,9 +2637,9 @@ } }, "@types/connect": { - "version": "3.4.34", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", - "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", "dev": true, "requires": { "@types/node": "*" @@ -2661,9 +2661,9 @@ "dev": true }, "@types/express": { - "version": "4.17.12", - "resolved": "https://nexus.tegonal.com/repository/npm-proxy/@types/express/-/express-4.17.12.tgz", - "integrity": "sha512-pTYas6FrP15B1Oa0bkN5tQMNqOcVXa9j4FTFtO8DWI9kppKib+6NJtfTOOLcwxuuYvcX2+dVG6et1SxW/Kc17Q==", + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", "dev": true, "requires": { "@types/body-parser": "*", @@ -2673,9 +2673,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.21", - "resolved": "https://nexus.tegonal.com/repository/npm-proxy/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz", - "integrity": "sha512-gwCiEZqW6f7EoR8TTEfalyEhb1zA5jQJnRngr97+3pzMaO1RKoI1w2bw07TK72renMUVWcWS5mLI6rk1NqN0nA==", + "version": "4.17.26", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", + "integrity": "sha512-zeu3tpouA043RHxW0gzRxwCHchMgftE8GArRsvYT0ByDMbn19olQHx5jLue0LxWY6iYtXb7rXmuVtSkhy9YZvQ==", "dev": true, "requires": { "@types/node": "*", @@ -2719,21 +2719,21 @@ "dev": true }, "@types/qs": { - "version": "6.9.6", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.6.tgz", - "integrity": "sha512-0/HnwIfW4ki2D8L8c9GVcG5I72s9jP5GSLVF0VIXDW00kmIpA6O33G7a8n59Tmh7Nz0WUC3rSb7PTY/sdW2JzA==", + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", "dev": true }, "@types/range-parser": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", - "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, "@types/serve-static": { - "version": "1.13.9", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.9.tgz", - "integrity": "sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==", + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", "dev": true, "requires": { "@types/mime": "^1", @@ -3170,16 +3170,16 @@ }, "dependencies": { "mime-db": { - "version": "1.40.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", - "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" }, "mime-types": { - "version": "2.1.24", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", - "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", "requires": { - "mime-db": "1.40.0" + "mime-db": "1.51.0" } } } @@ -4882,17 +4882,17 @@ "dev": true }, "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" }, "dependencies": { "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, @@ -5998,16 +5998,16 @@ } }, "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.2.tgz", + "integrity": "sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg==", "requires": { "accepts": "~1.3.7", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.19.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.4.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "~1.1.2", @@ -6021,23 +6021,93 @@ "on-finished": "~2.3.0", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.9.6", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", "statuses": "~1.5.0", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "dependencies": { + "body-parser": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.1.tgz", + "integrity": "sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA==", + "requires": { + "bytes": "3.1.1", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.6", + "raw-body": "2.4.2", + "type-is": "~1.6.18" + } + }, + "bytes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", + "integrity": "sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg==" + }, + "cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "qs": { + "version": "6.9.6", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", + "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==" + }, + "raw-body": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.2.tgz", + "integrity": "sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ==", + "requires": { + "bytes": "3.1.1", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" } } }, @@ -6388,9 +6458,9 @@ } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fragment-cache": { "version": "0.2.1", @@ -6981,9 +7051,9 @@ } }, "ipaddr.js": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", - "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "is-accessor-descriptor": { "version": "0.1.6", @@ -7885,12 +7955,11 @@ "integrity": "sha512-Gh39xwJwBKy0OvFmWfBs/vDO4Nl7JhnJtkqNP76OUinQz7BiMoszHYrIDHHAaqVl/QKVxCEy4ZxC/XZninu7nQ==" }, "node-cache": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-4.2.1.tgz", - "integrity": "sha512-BOb67bWg2dTyax5kdef5WfU3X8xu4wPg+zHzkvls0Q/QpYycIFRLEEIdAx9Wma43DxG6Qzn4illdZoYseKWa4A==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", "requires": { - "clone": "2.x", - "lodash": "^4.17.15" + "clone": "2.x" } }, "node-libs-browser": { @@ -8533,12 +8602,12 @@ "dev": true }, "proxy-addr": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", - "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.0" + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" } }, "prr": { @@ -9117,9 +9186,9 @@ } }, "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", "requires": { "debug": "2.6.9", "depd": "~1.1.2", @@ -9128,18 +9197,45 @@ "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "1.8.1", "mime": "1.6.0", - "ms": "2.1.1", + "ms": "2.1.3", "on-finished": "~2.3.0", "range-parser": "~1.2.1", "statuses": "~1.5.0" }, "dependencies": { + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" } } }, @@ -9150,14 +9246,14 @@ "dev": true }, "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.17.2" } }, "set-value": { diff --git a/package.json b/package.json index 37459e5a..af4b8e67 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "cookie-parser": "^1.4.5", "cors": "^2.8.5", "dotenv": "^10.0.0", - "express": "^4.17.1", + "express": "^4.17.2", "fs": "0.0.1-security", "haversine": "^1.1.1", "helmet": "^4.6.0", @@ -41,7 +41,7 @@ "js-md5": "^0.7.3", "lodash": "^4.17.21", "nocache": "^3.0.1", - "node-cache": "^4.2.1", + "node-cache": "^5.1.2", "query-overpass": "^1.5.5", "source-map-support": "^0.5.20", "swagger-express-middleware": "^4.0.2", @@ -51,7 +51,7 @@ "devDependencies": { "@types/cookie-parser": "^1.4.2", "@types/cors": "^2.8.10", - "@types/express": "^4.17.12", + "@types/express": "^4.17.13", "@types/geojson": "^7946.0.8", "@types/lodash": "^4.14.170", "@types/node": "^14.17.3", diff --git a/server/api/controllers/controller.ts b/server/api/controllers/controller.ts index 68b96576..9503d1ba 100644 --- a/server/api/controllers/controller.ts +++ b/server/api/controllers/controller.ts @@ -5,141 +5,93 @@ * and the profit contribution agreement available at https://www.my-d.org/ProfitContributionAgreement */ -import WikidataService from '../services/wikidata.service'; import l from '../../common/logger'; -import generateCityData from '../services/generateLocationData.service'; -import { cities, City, isCity, locationsCollection } from '../../../config/locations'; +import { locationsCollection } from '../../../config/locations'; import { fountain_property_metadata } from '../../../config/fountain.properties'; -import NodeCache from 'node-cache'; -import { essenceOf, fillWikipediaSummary } from '../services/processing.service'; -import { extractProcessingErrors } from './processing-errors.controller'; -import { getImageInfo, getImgsOfCat } from '../services/wikimedia.service'; -import { getCatExtract, getImgClaims } from '../services/claims.wm'; -import { isBlacklisted } from '../services/categories.wm'; -import { - MAX_IMG_SHOWN_IN_GALLERY, - LAZY_ARTIST_NAME_LOADING_i41db, //,CACHE_FOR_HRS_i45db -} from '../../common/constants'; + import sharedConstants from './../../common/shared-constants'; import { Request, Response } from 'express'; import { getSingleBooleanQueryParam, getSingleStringQueryParam } from './utils'; -import { Fountain, FountainCollection, GalleryValue, isDatabase } from '../../common/typealias'; -import { hasWikiCommonsCategories } from '../../common/wikimedia-types'; -import { ImageLike } from '../../../config/text2img'; - -// Configuration of Cache after https://www.npmjs.com/package/node-cache -const cityCache = new NodeCache({ - stdTTL: 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db, // time till cache expires, in seconds - checkperiod: 600, // how often to check for expiration, in seconds - default: 600 - deleteOnExpire: false, // on expire, we want the cache to be recreated not deleted - useClones: false, // do not create a clone of the data when fetching from cache -}); - -/* - * For each location (city), three JSON objects are created. Example for Zurich: - * - "ch-zh": contains the full data for all fountains of the location - * - "ch-zh_essential": contains a summary version of "ch-zh". This is the data loaded for display on the map. It is derived from "ch-zh". - * - "ch-zh_errors": contains a list of errors encountered when processing "ch-zh". - */ - -// when cached data expires, regenerate it (ignore non-essential) -cityCache.on('expired', key => { - // check if cache item key is neither the summary nor the list of errors. These will be updated automatically when the detailed city data are updated. - if (!key.includes('_essential') && !key.includes('_errors')) { - l.info(`controller.js cityCache.on('expired',...): Automatic cache refresh of ${key}`); - generateCityDataAndAddToCache(key, cityCache); - } -}); +import { BoundingBox, isDatabase, parseLngLat } from '../../common/typealias'; +import { + getFountainFromCacheIfNotForceRefreshOrFetch, + getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate, + populateCacheWithCities as populateLocationCacheWithCities, + getProcessingErrorsByBoundingBox, +} from '../services/generateLocationData.service'; +import { illegalState } from '../../common/illegalState'; +import { tileToLocationCacheKey } from '../services/locationCache'; export class Controller { constructor() { - // In production mode, process all fountains when starting the server so that the data are ready for the first requests + // In production mode, process all fountains of pre-defnied cities when starting the server + // so that the data is ready for the first requests if (process.env.NODE_ENV === 'production') { - cities.forEach(city => { - l.info(`controller.js Generating data for ${city}`); - generateCityData(city).then(fountainCollection => { - // save new data to storage - //TODO @ralfhauser, the old comment states // expire after two hours but CACHE_FOR_HRS_i45db is currently 48, which means after two days - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); - // create a reduced version of the data as well - cityCache.set(city + '_essential', essenceOf(fountainCollection)); - // also create list of processing errors (for proximap#206) - cityCache.set(city + '_errors', extractProcessingErrors(fountainCollection)); - }); + populateLocationCacheWithCities().catch((e: any) => { + l.error('unexpected error occured during populateLocationCacheWithCities\n' + e.stack); }); } } // Function to return detailed fountain information // When requesting detailed information for a single fountain, there are two types of queries - getSingle(req: Request, res: Response): void { - const queryType = getSingleStringQueryParam(req, 'queryType'); - const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true) ?? false; - - if (queryType === 'byId') { - l.info(`controller.js getSingle byId: refresh: ${refresh}`); - byId(req, res, refresh); - } else { - res.status(400).send('only byId supported'); - } + async getSingle(req: Request, res: Response, next: ErrorHandler): Promise { + handlingErrors(next, () => { + const queryType = getSingleStringQueryParam(req, 'queryType'); + const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true) ?? false; + + if (queryType === 'byId') { + l.info(`controller.js getSingle byId: refresh: ${refresh}`); + return this.byId(req, res, refresh); + } else { + illegalState('queryType only byId supported'); + } + }); } - - // Function to return all fountain information for a location. - byLocation(req: Request, res: Response): void { - const start = new Date(); - const city = getSingleStringQueryParam(req, 'city'); - if (!isCity(city)) { - throw Error('unsupported city given: ' + city); + private async byId(req: Request, res: Response, forceRefresh: boolean): Promise { + const loc = parseLngLat(getSingleStringQueryParam(req, 'loc')); + const database = getSingleStringQueryParam(req, 'database'); + if (!isDatabase(database)) { + illegalState('unsupported database given: ' + database); } - const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true); + const idval = getSingleStringQueryParam(req, 'idval'); - // if a refresh is requested or if no data is in the cache, then reprocess the fountains - if (refresh || cityCache.keys().indexOf(city) === -1) { - l.info(`controller.js byLocation: refresh: ${refresh} , city: ` + city); - generateCityData(city) - .then(fountainCollection => { - // save new data to storage - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); + const fountain = await getFountainFromCacheIfNotForceRefreshOrFetch(forceRefresh, database, idval, loc); + sendJson(res, fountain, 'byId'); + } - // create a reduced version of the data as well - const r_essential = essenceOf(fountainCollection); - cityCache.set(city + '_essential', r_essential); + async getByBounds(req: Request, res: Response, next: ErrorHandler): Promise { + handlingErrors(next, () => { + const boundingBox = this.getBoundingBoxFromQueryParam(req); + const essential = getSingleBooleanQueryParam(req, 'essential'); + const refresh = getSingleBooleanQueryParam(req, 'refresh'); - // return either the full or reduced version, depending on the "essential" parameter of the query - const essential = getSingleBooleanQueryParam(req, 'essential', /* isOptional = */ true) ?? false; - if (essential) { - sendJson(res, r_essential, 'r_essential'); - } else { - sendJson(res, fountainCollection, 'fountainCollection'); - } + return this.byBoundingBox(res, boundingBox, essential, refresh); + }); + } - // also create list of processing errors (for proximap#206) - cityCache.set(city + '_errors', extractProcessingErrors(fountainCollection)); - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; - l.info('controller.js byLocation generateLocationData: finished after ' + elapse.toFixed(1) + ' secs'); - }) - .catch(error => { - if (error.message) { - res.statusMessage = error.message; - } - res.status(500).send(error.stack); - }); - } - // otherwise, get the data from storage - else { - const essential = getSingleBooleanQueryParam(req, 'essential', /* isOptional = */ true) ?? false; - if (essential) { - sendJson(res, cityCache.get(city + '_essential'), 'fromCache essential'); - } else { - sendJson(res, cityCache.get(city), 'fromCache'); - } - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; - l.info('controller.js byLocation: finished after ' + elapse.toFixed(1) + ' secs'); - } + private getBoundingBoxFromQueryParam(req: Request): BoundingBox { + const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw')); + const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne')); + const boundingBox = BoundingBox(southWest, northEast); + return boundingBox; } + private async byBoundingBox( + res: Response, + boundingBox: BoundingBox, + essential: boolean, + forceRefresh: boolean + ): Promise { + const collection = await getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + boundingBox, + essential, + 'byBounds', + /* debugAll = */ false + ); + sendJson(res, collection, 'fountainCollection'); + } /** * Function to return metadata regarding all the fountain properties that can be displayed. * (e.g. name translations, definitions, contribution information and tips) @@ -168,26 +120,10 @@ export class Controller { * Function to extract processing errors from detailed list of fountains */ getProcessingErrors(req: Request, res: Response): void { - // returns all processing errors for a given location - // made for #206 - const city = getSingleStringQueryParam(req, 'city'); - const key = city + '_errors'; - - if (cityCache.keys().indexOf(key) < 0) { - // if data not in cache, create error list - cityCache.set(key, extractProcessingErrors(cityCache.get(city))); - } - cityCache.get(key, (err, value) => { - if (!err) { - sendJson(res, value, 'cityCache.get ' + key); - l.info('controller.js: getProcessingErrors !err sent'); - } else { - const errMsg = 'Error with cache: ' + err; - l.info('controller.js: getProcessingErrors ' + errMsg); - res.statusMessage = errMsg; - res.status(500).send(err.stack); - } - }); + // returns all processing errors for a given location made for #206 + const boundingBox = this.getBoundingBoxFromQueryParam(req); + const errors = getProcessingErrorsByBoundingBox(boundingBox); + sendJson(res, errors ?? [], tileToLocationCacheKey(boundingBox)); } } export const controller = new Controller(); @@ -195,22 +131,24 @@ export const controller = new Controller(); function sendJson(resp: Response, obj: Record | undefined, dbg: string): void { //TODO consider using https://github.com/timberio/timber-js/issues/69 or rather https://github.com/davidmarkclements/fast-safe-stringify try { - if (obj == undefined) { + if (obj === undefined) { l.error('controller.js doJson null == obj: ' + dbg); + resp.status(404).send(); + } else { + resp.json(obj); + //TODO @ralfhauser, neihter res.finish nor res.close exist, logging the json would need to be done before hand + // let res = resp.json(obj); + // if(process.env.NODE_ENV !== 'production') { + // // https://nodejs.org/dist/latest-v12.x/docs/api/http.html#http_class_http_serverresponse Event finish + // res.finish = res.close = function (event) { + // //not working :( https://github.com/water-fountains/datablue/issues/40 + // //https://github.com/expressjs/express/issues/4158 https://github.com/expressjs/express/blob/5.0/lib/response.js + // l.info('controller.js doJson length: keys '+Object.keys(obj).length+ + // //'\n responseData.data.length '+resp.responseData.data.length+ + // ' - '+dbg); + // } + // } } - resp.json(obj); - //TODO @ralfhauser, neihter res.finish nor res.close exist, logging the json would need to be done before hand - // let res = resp.json(obj); - // if(process.env.NODE_ENV !== 'production') { - // // https://nodejs.org/dist/latest-v12.x/docs/api/http.html#http_class_http_serverresponse Event finish - // res.finish = res.close = function (event) { - // //not working :( https://github.com/water-fountains/datablue/issues/40 - // //https://github.com/expressjs/express/issues/4158 https://github.com/expressjs/express/blob/5.0/lib/response.js - // l.info('controller.js doJson length: keys '+Object.keys(obj).length+ - // //'\n responseData.data.length '+resp.responseData.data.length+ - // ' - '+dbg); - // } - // } } catch (err: unknown) { const errS = 'controller.js doJson errors: "' + err + '" ' + dbg; l.error(errS); @@ -218,359 +156,9 @@ function sendJson(resp: Response, obj: Record | undefined, dbg: str } } -//TODO @ralfhauser, this function is too long and one does not have a good overview any more. Consider splitting it up into several functions -/** - * Function to respond to request by returning the fountain as defined by the provided identifier - */ -function byId(req: Request, res: Response, forceRefresh: boolean): Promise { - const city = getSingleStringQueryParam(req, 'city'); - if (!isCity(city)) { - return new Promise((_, reject) => reject('unsupported city given: ' + city)); - } - const database = getSingleStringQueryParam(req, 'database'); - if (!isDatabase(database)) { - return new Promise((_, reject) => reject('unsupported database given: ' + database)); - } - const idval = getSingleStringQueryParam(req, 'idval'); - const dbg = idval; - - let name = 'unkNamById'; - // l.info('controller.js byId: '+cityS+' '+dbg); - let fountainCollection = cityCache.get(city); - - // l.info('controller.js byId in promise: '+cityS+' '+dbg); - const cityPromises: Promise[] = []; - if (forceRefresh || fountainCollection === undefined) { - l.info('controller.js byId: ' + city + ' not found in cache ' + dbg + ' - start city lazy load'); - const genLocPrms = generateCityDataAndAddToCache(city, cityCache); - cityPromises.push(genLocPrms); - } - return Promise.all(cityPromises) - .then( - () => { - if (forceRefresh || fountainCollection === undefined) { - fountainCollection = cityCache.get(city); - } - if (fountainCollection !== undefined) { - const fountain = fountainCollection.features.find(f => f.properties['id_' + database]?.value === idval); - const imgMetaPromises: Promise[] = []; - let lazyAdded = 0; - const gl = -1; - if (fountain === undefined) { - l.info('controller.js byId: of ' + city + ' not found in cache ' + dbg); - return undefined; - } else { - const props = fountain.properties; - // l.info('controller.js byId fountain: '+cityS+' '+dbg); - if (null != props) { - name = props.name.value; - if (LAZY_ARTIST_NAME_LOADING_i41db) { - imgMetaPromises.push(WikidataService.fillArtistName(fountain, dbg)); - } - imgMetaPromises.push(WikidataService.fillOperatorInfo(fountain, dbg)); - fillWikipediaSummary(fountain, dbg, 1, imgMetaPromises); - const gallery = props.gallery; - // l.info('controller.js byId props: '+cityS+' '+dbg); - if (null != gallery && null != gallery.value) { - // l.info('controller.js byId gl: '+cityS+' '+dbg); - if (0 < gallery.value.length) { - // l.info('controller.js byId: of '+cityS+' found gal of size '+gl+' "'+name+'" '+dbg); - let i = 0; - let lzAtt = ''; - const showDetails = true; - const singleRefresh = true; - const imgUrlSet = new Set(); - const catPromises: Promise[] = []; - let numberOfCategories = -1; - let numberOfCategoriesLazyAdded = 0; - const imgUrlsLazyByCategory: ImageLike[] = []; - // TODO @ralfhauser, this condition does not make sense, if value.length < 0 means basically if it is empty and if it is empty, then numberOfCategories will always be 0 and the for-loop will do nothing - if (hasWikiCommonsCategories(props) && 0 < props.wiki_commons_name.value.length) { - numberOfCategories = props.wiki_commons_name.value.length; - let j = 0; - for (const cat of props.wiki_commons_name.value) { - j++; - if (null == cat) { - l.info(i + '-' + j + ' controller.js: null == commons category "' + cat + '" "' + dbg); - continue; - } - if (null == cat.c) { - l.info(i + '-' + j + ' controller.js: null == commons cat.c "' + cat + '" "' + dbg); - continue; - } - if (isBlacklisted(cat.c)) { - l.info(i + '-' + j + ' controller.js: commons category blacklisted "' + cat + '" "' + dbg); - continue; - } - const add = 0 > cat.l; - if (add) { - numberOfCategoriesLazyAdded++; - if (0 == imgUrlSet.size) { - for (const img of gallery.value) { - imgUrlSet.add(img.pgTit); - } - } - const catPromise = getImgsOfCat( - cat, - dbg, - city, - imgUrlSet, - imgUrlsLazyByCategory, - 'dbgIdWd', - props, - true - ); - //TODO we might prioritize categories with small number of images to have greater variety of images? - catPromises.push(catPromise); - } - getCatExtract(singleRefresh, cat, catPromises, dbg); - } - } - return Promise.all(catPromises).then( - r => { - for (let k = 0; k < imgUrlsLazyByCategory.length && k < MAX_IMG_SHOWN_IN_GALLERY; k++) { - //between 6 && 50 imgs are on the gallery-preview - const img = imgUrlsLazyByCategory[k]; - //TODO @ralfhauser, val does not exist on GalleryValue but value, changed it - const nImg: GalleryValue = { - s: img.src, - pgTit: img.value, - c: img.cat, - t: img.typ, - }; - gallery.value.push(nImg); - } - if (0 < imgUrlsLazyByCategory.length) { - l.info( - 'controller.js byId lazy img by lazy cat added: attempted ' + - imgUrlsLazyByCategory.length + - ' in ' + - numberOfCategoriesLazyAdded + - '/' + - numberOfCategories + - ' cats, tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - for (const img of gallery.value) { - const imMetaDat = img.metadata; - if (null == imMetaDat && 'wm' == img.t) { - lzAtt += i + ','; - l.info( - 'controller.js byId lazy getImageInfo: ' + - city + - ' ' + - i + - '/' + - gl + - ' "' + - img.pgTit + - '" "' + - name + - '" ' + - dbg - ); - imgMetaPromises.push( - getImageInfo( - img, - i + '/' + gl + ' ' + dbg + ' ' + name + ' ' + city, - showDetails, - props - ).catch(giiErr => { - //TODO @ralfhauser, dbgIdWd does not exist - const dbgIdWd = undefined; - l.info( - 'wikimedia.service.js: fillGallery getImageInfo failed for "' + - img.pgTit + - '" ' + - dbg + - ' ' + - city + - ' ' + - dbgIdWd + - ' "' + - name + - '"' + - '\n' + - giiErr.stack - ); - }) - ); - lazyAdded++; - } else { - // l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg); - } - getImgClaims(singleRefresh, img, imgMetaPromises, i + ': ' + dbg); - i++; - } - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading: attempted ' + - lazyAdded + - '/' + - gl + - ' (' + - lzAtt + - ') of ' + - city + - ' ' + - dbg + - ' "' + - name + - '"' - ); - } - return Promise.all(imgMetaPromises).then( - r => { - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading after promise: attempted ' + - lazyAdded + - ' tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - //TODO @ralfhauser this is a clear smell, we already send the response before we resolve the promise - // it would be better if we return the fountain in a then once the promise completes - sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); - l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); - return fountain; - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - } else { - l.info('controller.js byId: of ' + city + ' gl < 1 ' + dbg); - return Promise.all(imgMetaPromises).then( - r => { - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading after promise: attempted ' + - lazyAdded + - ' tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - //TODO @ralfhauser this is a clear smell, we already send the response before we resovle the promise - sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); - l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); - return fountain; - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - } - } else { - l.info('controller.js byId: of ' + city + ' gallery null || null == gal.value ' + dbg); - return undefined; - } - } else { - l.info('controller.js byId: of ' + city + ' no props ' + dbg); - return undefined; - } - } - } else { - return undefined; - } - // l.info('controller.js byId: end of '+cityS+' '+dbg); - }, - err => { - l.error(`controller.js byId: Failed on genLocPrms: ${err.stack} .` + dbg + ' ' + city); - return undefined; - } - ) - .catch(e => { - //TODO @ralfhauser, this error will never occurr because we already defined an error case two lines above - l.error(`controller.js byId: Error finding fountain in preprocessed data: ${e} , city: ` + city + ' ' + dbg); - l.error(e.stack); - return undefined; - }); +// TODO should no longer be necessary starting with Express 5.x +function handlingErrors(next: ErrorHandler, action: () => Promise) { + action().catch(next); } -export function generateCityDataAndAddToCache(city: City, cityCache: NodeCache): Promise { - // trigger a reprocessing of the location's data, based on the key. - const genLocPrms = generateCityData(city) - .then(fountainCollection => { - // save newly generated fountainCollection to the cache - let numberOfFountains = -1; - if (fountainCollection?.features != null) { - numberOfFountains = fountainCollection.features.length; - } - //TODO @ralfhauser, the old comment states // expire after two hours but CACHE_FOR_HRS_i45db is currently 48, which means after two days - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); // expire after two hours - - // create a reduced version of the data as well - const essence = essenceOf(fountainCollection); - cityCache.set(city + '_essential', essence); - let ess = -1; - if (null != essence && null != essence.features) { - ess = essence.features.length; - } - - // also create list of processing errors (for proximap#206) - const processingErrors = extractProcessingErrors(fountainCollection); - cityCache.set(city + '_errors', processingErrors); - //TODO @ralfhauser, processingErrors is never null but an array, which also means processingErrors.features never exists and hence this will always be false - // let prcErr = -1; - // if (null != processingErrors && null != processingErrors.features) { - // prcErr = processingErrors.features.length; - // } - const prcErr = processingErrors.length; - l.info( - `generateLocationDataAndCache setting cache of ${city} ` + - ' ftns: ' + - numberOfFountains + - ' ess: ' + - ess + - ' prcErr: ' + - prcErr - ); - return fountainCollection; - }) - .catch(error => { - l.error(`controller.js unable to set Cache. Error: ${error.stack}`); - // TODO @ralfhauser, return void is not so nice IMO but that's what was defined beforehand implicitly - return; - }); - return genLocPrms; -} +type ErrorHandler = (error: unknown) => unknown; diff --git a/server/api/controllers/processing-errors.controller.ts b/server/api/controllers/processing-errors.controller.ts index e945fe52..8f45afd9 100644 --- a/server/api/controllers/processing-errors.controller.ts +++ b/server/api/controllers/processing-errors.controller.ts @@ -10,12 +10,13 @@ import _ from 'lodash'; import { l } from '../../common/logger'; import { FountainCollection } from '../../common/typealias'; +import '../../common/importAllExtensions'; //TODO @ralfhauser, I don't know the type of errorCollection since I was not able to get a real example of an issue, please narrow down accordingly export type ProcessingError = any; export function hasProcessingIssues(obj: Record): obj is { issues: ProcessingError[] } { - return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues); + return Object.prototype.hasOwnProperty.call(obj, 'issues') && Array.isArray(obj.issues) && obj.issues.nonEmpty(); } export function extractProcessingErrors(fountainCollection: FountainCollection | undefined): ProcessingError[] { diff --git a/server/api/controllers/router.ts b/server/api/controllers/router.ts index 3d6ace0b..7213c5c3 100644 --- a/server/api/controllers/router.ts +++ b/server/api/controllers/router.ts @@ -12,10 +12,10 @@ import { buildInfoController } from './build-info.controller'; // This file maps API routes to functions export const Router = express .Router() - .get('/fountain/', controller.getSingle) - .get('/fountains/', controller.byLocation) - .get('/metadata/fountain_properties/', controller.getPropertyMetadata) - .get('/metadata/locations/', controller.getLocationMetadata) - .get('/metadata/shared-constants/', controller.getSharedConstants) - .get('/processing-errors/', controller.getProcessingErrors) + .get('/fountain', controller.getSingle.bind(controller)) + .get('/fountains', controller.getByBounds.bind(controller)) + .get('/metadata/fountain_properties', controller.getPropertyMetadata.bind(controller)) + .get('/metadata/locations', controller.getLocationMetadata.bind(controller)) + .get('/metadata/shared-constants', controller.getSharedConstants.bind(controller)) + .get('/processing-errors', controller.getProcessingErrors.bind(controller)) .get('/build-info', buildInfoController); diff --git a/server/api/controllers/utils.ts b/server/api/controllers/utils.ts index 7ee1648d..652f065e 100644 --- a/server/api/controllers/utils.ts +++ b/server/api/controllers/utils.ts @@ -116,7 +116,9 @@ function typeCheckAndConvertParam( } else if (typeCheck(param)) { return typeConversion(param); } else { - throw Error(`${paramName} was of a wrong type, expected ${type} was ${JSON.stringify(param)} ${typeof param}`); + throw Error( + `${paramName} was of a wrong type, expected ${type} was ${JSON.stringify(param)} with type ${typeof param}` + ); } } diff --git a/server/api/services/database.service.ts b/server/api/services/database.service.ts deleted file mode 100644 index 7657683a..00000000 --- a/server/api/services/database.service.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * @license - * (c) Copyright 2019 - 2020 | MY-D Foundation | Created by Matthew Moy de Vitry - * Use of this code is governed by the GNU Affero General Public License (https://www.gnu.org/licenses/agpl-3.0) - * and the profit contribution agreement available at https://www.my-d.org/ProfitContributionAgreement - */ - -import { essenceOf } from './processing.service'; -//TODO @ralfhauser, CACHE_FOR_HRS_i45db is not defined in constants, please adjust this number -// import {CACHE_FOR_HRS_i45db} from "../../common/constants"; -const CACHE_FOR_HRS_i45db = 1; -import l from '../../common/logger'; -import { generateCityDataAndAddToCache } from '../controllers/controller'; -import _ from 'lodash'; -import haversine from 'haversine'; -import NodeCache from 'node-cache'; -import {} from 'geojson'; -import { Fountain, FountainCollection } from '../../common/typealias'; -import { City } from '../../../config/locations'; - -export function updateCacheWithFountain(cache: NodeCache, fountain: Fountain, city: City): Fountain { - // updates cache and returns fountain with datablue id - // get city data from cache - let fountains = cache.get(city); - const cacheTimeInSecs = 60 * 60 * CACHE_FOR_HRS_i45db; - if (!fountains && !city.includes('_essential') && !city.includes('_errors')) { - l.info( - `updateCacheWithFountain server-side city data disappeared (server restart?) - cache recreation for ${city}` - ); - generateCityDataAndAddToCache(city, cache); - //TODO @ralf.hauser, this is buggy as generateCityDataAndAddToCache returns a promise and most likely did not finish at this point - fountains = cache.get(city); - } - if (fountains) { - // replace fountain - [fountains, fountain] = replaceFountain(fountains, fountain, city); - // send to cache - //TODO consider whether really to fully extend the cache-time for the whole city just because one fountain was refreshed - // a remaining city-cache-time could be calculated with getTtl(cityname) - cache.set(city, fountains, cacheTimeInSecs); - // create a reduced version of the data as well - const r_essential = essenceOf(fountains); - cache.set(city + '_essential', r_essential, cacheTimeInSecs); - return fountain; - } - l.info( - 'database.services.js updateCacheWithFountain: no fountains were in cache of city ' + - city + - ' tried to work on ' + - fountain - ); - return fountain; -} - -function replaceFountain( - fountains: FountainCollection, - fountain: Fountain, - cityName: string -): [FountainCollection, Fountain] { - // update cache with fountain and assign correct datablue id - - const distances: [number, Fountain, number][] = []; - - for (let i = 0; i < fountains.features.length; i++) { - if (isMatch(fountains.features[i], fountain)) { - //replace fountain - fountain.properties.id = fountains.features[i].properties.id; - fountains.features[i] = fountain; - l.info('database.services.js replaceFountain: ismatch ftn ' + i + ', city ' + cityName + ' , ftn ' + fountain); - return [fountains, fountain]; - } else { - // compute distance otherwise - distances.push([ - i, - fountains.features[i], - haversine(fountains.features[i].geometry.coordinates, fountain.geometry.coordinates, { - unit: 'meter', - format: '[lon,lat]', - }), - ]); - } - } - - const triple = _.minBy(distances, p => p[2]); - if (triple !== undefined && triple[2] < 15) { - //TODO @ralf.hauser `f` did not exist here. I assumed that the fountain should be replaced by the fountain which is nearest. Please verify this change is correct - const [index, nearestFountain, distance] = triple; - //replace fountain - // fountain.properties.id = f.properties.id; - l.info( - 'database.services.js replaceFountain: replaced with distance ' + - distance + - ', city ' + - cityName + - ' , ftn ' + - fountain - ); - fountain.properties.id = nearestFountain.properties.id; - fountains.features[index] = fountain; - return [fountains, fountain]; - } else { - // fountain was not found; just add it to the list - fountain.properties.id = _.max(fountains.features.map(f => f.properties.id)) + 1; - fountains.features.push(fountain); - l.info( - 'database.services.js replaceFountain: added with distance ' + - triple?.[2] + - ', city ' + - cityName + - ' , ftn ' + - fountain - ); - return [fountains, fountain]; - } -} - -function isMatch(f1: Fountain, f2: Fountain): boolean { - // returns true if match, otherwise returns distance - return ['id_wikidata', 'id_osm'].some(idName => { - f1.properties && f2.properties && f1.properties[idName].value === f2.properties[idName].value; - }); -} diff --git a/server/api/services/generateLocationData.service.ts b/server/api/services/generateLocationData.service.ts index bb84e4b8..9617652a 100644 --- a/server/api/services/generateLocationData.service.ts +++ b/server/api/services/generateLocationData.service.ts @@ -6,7 +6,7 @@ */ import l from '../../common/logger'; -import { City, locationsCollection } from '../../../config/locations'; +import { cities, getCityBoundingBox } from '../../../config/locations'; import OsmService from '../services/osm.service'; import WikidataService from '../services/wikidata.service'; import { conflate } from '../services/conflate.data.service'; @@ -14,133 +14,473 @@ import applyImpliedPropertiesOsm from '../services/applyImplied.service'; import { createUniqueIds, defaultCollectionEnhancement, + essenceOf, fillInMissingWikidataFountains, + fillWikipediaSummary, } from '../services/processing.service'; -import { FountainCollection } from '../../common/typealias'; -import { MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; +import { + BoundingBox, + Database, + Fountain, + FountainCollection, + GalleryValue, + LngLat, + positionToLngLat, +} from '../../common/typealias'; +import { hasWikiCommonsCategories, MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; +import sharedConstants from '../../common/shared-constants'; +import { extractProcessingErrors, ProcessingError } from '../controllers/processing-errors.controller'; +import { illegalState } from '../../common/illegalState'; +import '../../common/importAllExtensions'; +import { + cacheEssentialFountainCollection, + cacheFullFountainCollection, + cacheProcessingErrors, + getBoundingBoxOfTiles, + getCachedEssentialFountainCollection, + getCachedFullFountainCollection, + getCachedProcessingErrors, + getTileOfLocation, + splitInTiles, + Tile, + tileToLocationCacheKey, +} from './locationCache'; +import { sleep } from '../../common/sleep'; +import { LAZY_ARTIST_NAME_LOADING_i41db, MAX_IMG_SHOWN_IN_GALLERY } from '../../common/constants'; +import { ImageLike } from '../../../config/text2img'; +import { isBlacklisted } from './categories.wm'; +import { getImageInfo, getImgsOfCat } from './wikimedia.service'; +import { getCatExtract, getImgClaims } from './claims.wm'; -/** - * This function creates fountain collections - * @param {string} locationName - the code name of the location for which fountains should be processed - */ -function generateCityData(locationName: City): Promise { - const start = new Date(); - l.info(`generateLocationData.service.js: processing all fountains from "${locationName}" `); - - return new Promise((resolve, reject) => { - const logAndRejectError = function (err: string) { - l.error(err); - reject(new Error(err)); - }; - - // get bounding box of location - const location = locationsCollection[locationName]; - if (location == undefined) { - logAndRejectError(`location not found in config: ${locationName}`); - } else { - //TODO @ralfhauser the following checks are unnecessary IMO as they cannot be null according to the definition - const bbox = location.bounding_box; - if (null == bbox) { - const err = `fatal: null == bbox for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.latMin) { - const err = `fatal: null == bbox.latMin for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.lngMin) { - const err = `fatal: null == bbox.lngMin for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.latMax) { - const err = `fatal: null == bbox.latMax for ${locationName}`; - l.error(err); - reject(new Error(err)); +//TODO @ralf.hauser reconsider this functionality. If a user queries a city in the same time then it is more likely +//that we run into a throttling timeout. Maybe only load the default city? +export async function populateCacheWithCities(): Promise { + for (const city of cities) { + if (city !== 'test') continue; + l.info(`Generating data for ${city}`); + await getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + /*forceRefresh= */ false, + getCityBoundingBox(city), + /* essential= */ false, + /* dbg=*/ city, + /* debugAll= */ false, + sharedConstants.CITY_TTL_IN_HOURS + ).catch(async (e: any) => { + // we still want to try to populate the others, thus we are not re-throwing. + //TODO @ralf.hauser it would actually be better if we react to a 429 response from OSM or wikidata + if (e.statusCode === 429) { + console.error('we got throttled, waiting for 2 minutes'); + // wait 2 minutes because OSM or wikidata throttled us + await sleep(2 * 60 * 1000); + } else { + console.error(e.message + '\n' + e.stack); } - if (null == bbox.lngMax) { - const err = `fatal: null == bbox.lngMax for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (bbox.lngMin > bbox.lngMax) { - const err = `fatal: bbox.lngMin > bbox.lngMax for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (bbox.latMin > bbox.latMax) { - const err = `fatal: bbox.latMin > bbox.latMax for ${locationName}`; - l.error(err); - reject(new Error(err)); + }); + // wait 30 seconds between each city to lower the chance that OSM or wikidata throttles us + await sleep(30 * 1000); + } + l.info('finished populating cache with cities'); +} + +export async function getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + forceRefresh: boolean, + boundingBox: BoundingBox, + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined = undefined +): Promise { + const start = new Date(); + const tiles = splitInTiles(boundingBox); + l.info('processing ' + tiles.length + ' tiles'); + const collection = await byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + tiles, + essential, + dbg, + debugAll, + ttlInHours + ); + + const end = new Date(); + const elapse = (end.getTime() - start.getTime()) / 1000; + l.info( + 'generateLocationData.service.js: after ' + + elapse.toFixed(1) + + ' secs successfully processed all (size ' + + collection.features.length + + `) fountains from ${dbg} \nstart: ` + + start.toISOString() + + '\nend: ' + + end.toISOString() + ); + return collection; +} +export function getProcessingErrorsByBoundingBox(boundingBox: BoundingBox): ProcessingError[] { + const arr = splitInTiles(boundingBox).map(tile => getCachedProcessingErrors(tile)); + return arr.reduce((acc, v) => (v !== undefined ? acc.concat(v.value) : acc), /*initial=*/ []); +} + +async function byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh: boolean, + tiles: BoundingBox[], + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined = undefined +): Promise { + type AccType = [Fountain[], Date] | undefined; + + //TODO @ralf.hauser, we could optimise this a bit in case only bounding boxes at the border are not cached yet + const maybeArr = forceRefresh + ? undefined // don't even search in cache in case of forceRefresh + : tiles.reduce( + (acc, tile) => { + if (acc === undefined) { + // previous was not in cache, no need to search further + return undefined; + } else { + const cacheEntry = essential + ? getCachedEssentialFountainCollection(tile) + : getCachedFullFountainCollection(tile); + if (cacheEntry !== undefined) { + const [fountains, currentLastScan] = acc; + const lastScan = cacheEntry.value.last_scan; + const olderLastScan = lastScan && lastScan < currentLastScan ? lastScan : currentLastScan; + return [fountains.concat(cacheEntry.value.features), olderLastScan] as AccType; + } else { + return undefined; + } + } + }, + /* initial= */ [[], new Date()] as AccType + ); + + if (maybeArr !== undefined) { + const [fountains, lastScan] = maybeArr; + l.info('all tiles in cache'); + // all in cache, return immediately + return Promise.resolve(FountainCollection(fountains, lastScan)); + } else { + const lastScan = new Date(); + const fountains = await fetchFountainsFromServerAndUpdateCache( + tiles, + essential, + dbg, + debugAll, + ttlInHours, + lastScan + ); + return FountainCollection(fountains, lastScan); + } +} + +async function fetchFountainsFromServerAndUpdateCache( + tiles: Tile[], + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined, + lastScan: Date +): Promise { + const boundingBox = getBoundingBoxOfTiles(tiles); + const fountains = await fetchFountainsByBoundingBox(boundingBox, dbg, debugAll); + + const groupedByTile = fountains.groupBy(fountain => + tileToLocationCacheKey(getTileOfLocation(positionToLngLat(fountain.geometry.coordinates))) + ); + + const collections = tiles.map(tile => { + const cacheKey = tileToLocationCacheKey(tile); + const fountains = groupedByTile.get(cacheKey) ?? []; + let fountainCollection: FountainCollection | undefined = FountainCollection(fountains, lastScan); + updateCacheWithFountains(tile, fountainCollection, ttlInHours); + fountainCollection = ( + essential ? getCachedEssentialFountainCollection(tile) : getCachedFullFountainCollection(tile) + )?.value; + if (fountainCollection === undefined) { + illegalState(`fountainCollection ${cacheKey} was undefined after writing it to the cache`); + } + return fountainCollection; + }); + + return collections.reduce((acc, collection) => acc.concat(collection.features), /* initial= */ new Array()); +} + +function fetchFountainsByBoundingBox(boundingBox: BoundingBox, dbg: string, debugAll: boolean): Promise { + const osmPromise = OsmService.byBoundingBox(boundingBox) + .then(arr => applyImpliedPropertiesOsm(arr)) + .catch(e => { + if ('getaddrinfo' == e.syscall) { + l.info('Are you offline from the internet?'); } + l.error( + `generateLocationDataService: Error collecting OSM data - generateLocationData: ${e.message}` + + ' latMin ' + + boundingBox.min.lat + + ', lngMim ' + + boundingBox.min.lng + + ', latMax ' + + boundingBox.max.lat + + ', lngMax ' + + boundingBox.max.lng + ); + throw e; + }); + + // get data from Wikidata + const wikidataPromise: Promise = WikidataService.idsByBoundingBox(boundingBox).then(r => + // TODO @ralf.hauser why not fetch the wikidata already in idsByBoundingBox? + WikidataService.byIds(r, dbg) + ); + + // conflate + return ( + Promise.all([osmPromise, wikidataPromise]) + // get any missing wikidata fountains for proximap#212 + .then(arr => fillInMissingWikidataFountains(arr[0], arr[1], dbg)) + .then(arr => conflate(arr, dbg, debugAll)) + .then(arr => defaultCollectionEnhancement(arr, dbg, debugAll)) + //TODO @ralf.hauser really required? + .then(arr => createUniqueIds(arr)) + ); +} + +function updateCacheWithFountains(tile: Tile, fountainCollection: FountainCollection, ttlInHours: number | undefined) { + // save newly generated fountainCollection to the cache + + const existing = getCachedFullFountainCollection(tile); + const ttl = 60 * 60 * (ttlInHours ?? existing?.ttl ?? sharedConstants.BOUNDING_BOX_TTL_IN_HOURS); + cacheFullFountainCollection(tile, fountainCollection, ttl); + + // create a reduced version of the data as well + const essence = essenceOf(fountainCollection); + cacheEssentialFountainCollection(tile, essence, ttl); + + // also create list of processing errors (for proximap#206) + const processingErrors = extractProcessingErrors(fountainCollection); + cacheProcessingErrors(tile, processingErrors, ttl); + //TODO @ralfhauser, processingErrors is never null but an array, which also means processingErrors.features never exists and hence this will always be false + // let prcErr = -1; + // if (null != processingErrors && null != processingErrors.features) { + // prcErr = processingErrors.features.length; + // } + l.info( + `generateLocationDataAndCache setting cache of ${tileToLocationCacheKey(tile)}` + + ' number of fountains: ' + + (fountainCollection?.features?.length ?? 'unknown') + + ' ess: ' + + (essence?.features?.length ?? 'unknown') + + ' prcErr: ' + + processingErrors.length + ); +} + +export async function getFountainFromCacheIfNotForceRefreshOrFetch( + forceRefresh: boolean, + database: Database, + idval: string, + loc: LngLat +): Promise { + const tile = getTileOfLocation(loc); + const collection = await byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + [tile], + /* essential = */ false, + 'database: ' + database + ' idval: ' + idval, + /* debugAll =*/ false + ); + const fountain = collection.features.find(f => f.properties['id_' + database]?.value === idval); + // TODO @ralf.hauser IMO it would make sense to distinguish this also in typing + const enrichedFountain = fountain ? enrichFountain(fountain, idval) : undefined; + if (enrichedFountain === undefined) { + l.info(`byId: loc ${loc.lat},${loc.lng} not in cache after loading, id/loc mismatch?`); + } + return enrichedFountain; +} + +//TODO @ralf.hauser this function is still very very smelly. I through now an error instead of not responding at all +function enrichFountain(fountain: Fountain, dbg: string): Promise { + const imgMetaPromises: Promise[] = []; + let lazyAdded = 0; + let gl = -1; + const props = fountain.properties; + if (null == props) illegalState('properties of fountain where undefined', fountain); - // get data from Osm - const osmPromise = OsmService.byBoundingBox(bbox.latMin, bbox.lngMin, bbox.latMax, bbox.lngMax) - .then(r => applyImpliedPropertiesOsm(r)) - .catch(e => { - if ('getaddrinfo' == e.syscall) { - l.info('Are you offline from the internet?'); + const name = props.name.value; + if (LAZY_ARTIST_NAME_LOADING_i41db) { + imgMetaPromises.push(WikidataService.fillArtistName(fountain, dbg)); + } + imgMetaPromises.push(WikidataService.fillOperatorInfo(fountain, dbg)); + fillWikipediaSummary(fountain, dbg, 1, imgMetaPromises); + + const gallery = props.gallery; + const galleryArr = gallery?.value; + if (!Array.isArray(galleryArr)) { + illegalState('controller.js byId: gallery null || null == gal.value || !isArray ' + dbg); + } + + if (galleryArr.isEmpty()) { + gl = galleryArr.length; + let i = 0; + let lzAtt = ''; + const showDetails = true; + const singleRefresh = true; + const imgUrlSet = new Set(); + const catPromises: Promise[] = []; + let numberOfCategories = -1; + let numberOfCategoriesLazyAdded = 0; + const imgUrlsLazyByCategory: ImageLike[] = []; + // TODO @ralfhauser, this condition does not make sense, if value.length < 0 means basically if it is empty and if it is empty, then numberOfCategories will always be 0 and the for-loop will do nothing + if (hasWikiCommonsCategories(props) && 0 < props.wiki_commons_name.value.length) { + numberOfCategories = props.wiki_commons_name.value.length; + let j = 0; + for (const cat of props.wiki_commons_name.value) { + j++; + if (null == cat) { + l.info(i + '-' + j + ' controller.js: null == commons category "' + cat + '" "' + dbg); + continue; + } + if (null == cat.c) { + l.info(i + '-' + j + ' controller.js: null == commons cat.c "' + cat + '" "' + dbg); + continue; + } + if (isBlacklisted(cat.c)) { + l.info(i + '-' + j + ' controller.js: commons category blacklisted "' + cat + '" "' + dbg); + continue; + } + const add = 0 > cat.l; + if (add) { + numberOfCategoriesLazyAdded++; + if (0 == imgUrlSet.size) { + for (const img of gallery.value) { + imgUrlSet.add(img.pgTit); + } } - l.error( - `generateLocationDataService.js: Error collecting OSM data - generateLocationData: ${e.stack} ` + - ' latMi ' + - bbox.latMin + - ', lngMi ' + - bbox.lngMin + - ', latMx ' + - bbox.latMax + - ', lngMx ' + - bbox.lngMax + const catPromise = getImgsOfCat(cat, dbg, imgUrlSet, imgUrlsLazyByCategory, 'dbgIdWd', props, true); + //TODO we might prioritize categories with small number of images to have greater variety of images? + catPromises.push(catPromise); + } + getCatExtract(singleRefresh, cat, catPromises, dbg); + } + } + return Promise.all(catPromises).then( + r => { + for (let k = 0; k < imgUrlsLazyByCategory.length && k < MAX_IMG_SHOWN_IN_GALLERY; k++) { + //between 6 && 50 imgs are on the gallery-preview + const img = imgUrlsLazyByCategory[k]; + //TODO @ralfhauser, val does not exist on GalleryValue but value, changed it + const nImg: GalleryValue = { + s: img.src, + pgTit: img.value, + c: img.cat, + t: img.typ, + }; + gallery.value.push(nImg); + } + if (0 < imgUrlsLazyByCategory.length) { + l.info( + 'controller.js byId lazy img by lazy cat added: attempted ' + + imgUrlsLazyByCategory.length + + ' in ' + + numberOfCategoriesLazyAdded + + '/' + + numberOfCategories + + ' cats, tot ' + + gl + + ' ' + + dbg + + ' "' + + name + + '" ' + + r.length ); - //TODO @ralfhauser, smelly, using the reject from an outer Promise. IMO better throw the exception - // reject(e); - throw e; - }); - - // get data from Wikidata - const wikidataPromise: Promise = WikidataService.idsByBoundingBox( - bbox.latMin, - bbox.lngMin, - bbox.latMax, - bbox.lngMax, - locationName - ).then(r => WikidataService.byIds(r, locationName)); - - const debugAll = -1 != locationName.indexOf('test'); - - // conflate - Promise.all([osmPromise, wikidataPromise]) - // get any missing wikidata fountains for proximap#212 - .then(r => fillInMissingWikidataFountains(r[0], r[1], locationName)) - .then(r => conflate(r, locationName, debugAll)) - .then(r => defaultCollectionEnhancement(r, locationName, debugAll)) - .then(r => createUniqueIds(r)) - .then(r => { - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; + } + for (const img of gallery.value) { + const imMetaDat = img.metadata; + if (null == imMetaDat && 'wm' == img.t) { + lzAtt += i + ','; + l.info( + 'controller.js byId lazy getImageInfo: ' + i + '/' + gl + ' "' + img.pgTit + '" "' + name + '" ' + dbg + ); + imgMetaPromises.push( + getImageInfo(img, i + '/' + gl + ' ' + dbg + ' ' + name, showDetails, props).catch(giiErr => { + //TODO @ralfhauser, dbgIdWd does not exist + const dbgIdWd = undefined; + l.info( + 'wikimedia.service.js: fillGallery getImageInfo failed for "' + + img.pgTit + + '" ' + + dbg + + ' ' + + dbgIdWd + + ' "' + + name + + '"' + + '\n' + + giiErr.stack + ); + }) + ); + lazyAdded++; + } else { + // l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg); + } + getImgClaims(singleRefresh, img, imgMetaPromises, i + ': ' + dbg); + i++; + } + if (0 < lazyAdded) { l.info( - 'generateLocationData.service.js: after ' + - elapse.toFixed(1) + - ' secs successfully processed all (size ' + - r.length + - `) fountains from ${locationName} \nstart: ` + - start.toISOString() + - '\nend: ' + - end.toISOString() + 'controller.js byId lazy img metadata loading: attempted ' + + lazyAdded + + '/' + + gl + + ' (' + + lzAtt + + ') ' + + dbg + + ' "' + + name + + '"' ); - resolve({ - type: 'FeatureCollection', - features: r, - }); - }) - .catch(err => { - l.error('generateLocationData.service.js - Promise.all([osmPromise, wikidataPromise]): ' + err.stack); - reject(err); - }); - } - }); + } + return waitForImgMetaPromises(fountain, lazyAdded, imgMetaPromises, gl + ' ' + dbg + ' "' + name + '"'); + }, + err => { + l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '"'); + throw err; + } + ); + } else { + l.info('controller.js byId: gl > 0 ' + dbg); + return waitForImgMetaPromises(fountain, lazyAdded, imgMetaPromises, gl + ' ' + dbg + ' "' + name + '"'); + } } -export default generateCityData; +function waitForImgMetaPromises( + fountain: Fountain, + lazyAdded: number, + imgMetaPromises: Promise[], + dbg: string +): Promise { + return Promise.all(imgMetaPromises).then( + r => { + if (0 < lazyAdded) { + l.info( + 'controller.js byId lazy img metadata loading after promise: attempted ' + + lazyAdded + + ' tot ' + + dbg + + '" ' + + r.length + ); + } + l.info('controller.js byId: res.json ' + dbg); + return fountain; + }, + err => { + l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg); + throw err; + } + ); +} diff --git a/server/api/services/google.service.ts b/server/api/services/google.service.ts index c2f73a98..6798e966 100644 --- a/server/api/services/google.service.ts +++ b/server/api/services/google.service.ts @@ -19,10 +19,12 @@ export interface Image { export function getStaticStreetView(fountain: Fountain): Promise { return new Promise(resolve => { + const lat = fountain.geometry.coordinates[1]; + const lng = fountain.geometry.coordinates[0]; resolve({ - big: `//maps.googleapis.com/maps/api/streetview?size=1200x600&location=${fountain.geometry.coordinates[1]},${fountain.geometry.coordinates[0]}&fov=120&key=${process.env.GOOGLE_API_KEY}`, - medium: `//maps.googleapis.com/maps/api/streetview?size=600x300&location=${fountain.geometry.coordinates[1]},${fountain.geometry.coordinates[0]}&fov=120&key=${process.env.GOOGLE_API_KEY}`, - small: `//maps.googleapis.com/maps/api/streetview?size=120x100&location=${fountain.geometry.coordinates[1]},${fountain.geometry.coordinates[0]}&fov=120&key=${process.env.GOOGLE_API_KEY}`, + big: `//maps.googleapis.com/maps/api/streetview?size=1200x600&location=${lat},${lng}&fov=120&key=${process.env.GOOGLE_API_KEY}`, + medium: `//maps.googleapis.com/maps/api/streetview?size=600x300&location=${lat},${lng}&fov=120&key=${process.env.GOOGLE_API_KEY}`, + small: `//maps.googleapis.com/maps/api/streetview?size=120x100&location=${lat},${lng}&fov=120&key=${process.env.GOOGLE_API_KEY}`, description: 'Google Street View and contributors', source_name: 'Google Street View', source_url: '//google.com', diff --git a/server/api/services/locationCache.ts b/server/api/services/locationCache.ts new file mode 100644 index 00000000..fa08fa62 --- /dev/null +++ b/server/api/services/locationCache.ts @@ -0,0 +1,155 @@ +import l from '../../common/logger'; +import NodeCache from 'node-cache'; +import sharedConstants from '../../common/shared-constants'; +import { BoundingBox, FountainCollection, LngLat, parseLngLat } from '../../common/typealias'; +import { ProcessingError } from '../controllers/processing-errors.controller'; +import { getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate } from './generateLocationData.service'; +import { illegalState } from '../../common/illegalState'; + +interface CacheEntry { + value: T; + ttl: number; +} + +// Configuration of Cache after https://www.npmjs.com/package/node-cache +const locationCache = new NodeCache({ + stdTTL: 60 * 60 * sharedConstants.BOUNDING_BOX_TTL_IN_HOURS, // time till cache expires, in seconds + checkperiod: 60 * 15, // how often to check for expiration, in seconds - default: 600 + deleteOnExpire: false, // on expire, we want the cache to be recreated not deleted + useClones: false, // do not create a clone of the data when fetching from cache (because we modify the data I guess) +}); + +export function getCachedFullFountainCollection(tile: Tile): CacheEntry | undefined { + return getCachedFountainCollection(tile, ''); +} +export function getCachedEssentialFountainCollection(tile: Tile): CacheEntry | undefined { + return getCachedFountainCollection(tile, ESSENTIAL_SUFFIX); +} +function getCachedFountainCollection(tile: Tile, suffix: string): CacheEntry | undefined { + return locationCache.get>(tileToLocationCacheKey(tile) + suffix); +} + +export function getCachedProcessingErrors(tile: Tile): CacheEntry | undefined { + return locationCache.get>(tileToLocationCacheKey(tile) + PROCESSING_ERRORS_SUFFIX); +} + +export function cacheFullFountainCollection(tile: Tile, fountainCollection: FountainCollection, ttl: number): void { + cacheEntry(tile, '', fountainCollection, ttl); +} +export function cacheEssentialFountainCollection( + tile: Tile, + fountainCollection: FountainCollection, + ttl: number +): void { + cacheEntry(tile, ESSENTIAL_SUFFIX, fountainCollection, ttl); +} + +export function cacheProcessingErrors(tile: Tile, errors: ProcessingError[], ttl: number): void { + cacheEntry(tile, PROCESSING_ERRORS_SUFFIX, errors, ttl); +} + +function cacheEntry(tile: Tile, suffix: string, entry: T, ttl: number): void { + locationCache.set>(tileToLocationCacheKey(tile) + suffix, { + value: entry, + ttl: ttl, + }); +} + +//TODO @ralf.hauser, check if it is realy worth it to store essential data +/* + * For each bounding box, 3 JSON objects are created. Example for Zurich: + * - "minLat,minLng:maxLat,maxLng": contains the full data for all fountains within the bounding box + * - "minLat,minLng:maxLat,maxLng_essential": contains the essential data for all fountains within the bounding box. This is the data loaded for display on the map. It is derived from the full data and cached additionally to speed up time + * - "minLat,minLng:maxLat,maxLng_errors": contains a list of errors encountered when processing the fountains within the bounding box + */ +// when cached data expires, regenerate full data (ignore expiration of essential and error data) +locationCache.on('expired', (key: string, cacheEntry: CacheEntry) => { + // check if cache item key is neither the summary nor the list of errors. These will be updated automatically when the detailed city data are updated. + if (isFullDataKey(key)) { + l.info(`controller locationCache.on('expired',...): Automatic cache refresh of ${key}`); + const tile = locationCacheKeyToTile(key); + getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + /*forceRefresh= */ true, + tile, + /* essential= */ false, + 'cache expired', + /* debugAll= */ false, + cacheEntry.ttl + ); + } +}); + +const ESSENTIAL_SUFFIX = '_essential'; +const PROCESSING_ERRORS_SUFFIX = '_errors'; +function isFullDataKey(key: string) { + return !key.endsWith(ESSENTIAL_SUFFIX) && !key.endsWith(PROCESSING_ERRORS_SUFFIX); +} + +export function locationCacheKeyToTile(key: string): Tile { + const minMax = key.split(':'); + return BoundingBox(parseLngLat(minMax[0]), parseLngLat(minMax[1])); +} + +export function tileToLocationCacheKey(tile: Tile): string { + return ( + `${tile.min.lat.toFixed(LNG_LAT_STRING_PRECISION)},${tile.min.lng.toFixed(LNG_LAT_STRING_PRECISION)}` + + ':' + + `${tile.max.lat.toFixed(LNG_LAT_STRING_PRECISION)},${tile.max.lng.toFixed(LNG_LAT_STRING_PRECISION)}` + ); +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +// 0.05 lat is ~5km +const TILE_SIZE = 0.05; +const ROUND_FACTOR = 20; // 1/0.05; +export const LNG_LAT_STRING_PRECISION = 2; +function roundToTilePrecision(n: number): number { + return Math.floor(n * ROUND_FACTOR) / ROUND_FACTOR; +} + +export function getTileOfLocation(lngLat: LngLat): Tile { + // for simplicity reasons we don't take the earth shape into account and + // act as if lng have everywhere the same distance (on all lat) + // + // one tile is 0.1 lat x 0.1 lng + // so the first tile is kind of at 0 - 0.1 lat and 0 - 0.1 lng + const lng = roundToTilePrecision(lngLat.lng); + const lat = roundToTilePrecision(lngLat.lat); + return Tile(lng, lat); +} + +export type Tile = BoundingBox; +export function Tile(lngMin: number, latMin: number): BoundingBox { + const maxLng = lngMin + TILE_SIZE; + const maxLat = latMin + TILE_SIZE; + return BoundingBox(LngLat(lngMin, latMin), LngLat(maxLng, maxLat)); +} + +export function splitInTiles(boundingBox: BoundingBox): Tile[] { + const startTile = getTileOfLocation(boundingBox.min); + const tiles = new Array(); + + for (let lng = startTile.min.lng; lng <= boundingBox.max.lng; lng += TILE_SIZE) { + for (let lat = startTile.min.lat; lat <= boundingBox.max.lat; lat += TILE_SIZE) { + tiles.push(Tile(lng, lat)); + } + } + return tiles; +} + +export function getBoundingBoxOfTiles(tiles: Tile[]): BoundingBox { + if (tiles.length === 0) illegalState('tiles was empty'); + else if (tiles.length === 1) { + return tiles[0]; + } else { + const lngs = tiles.map(x => x.min.lng).sort((a, b) => a - b); + const lats = tiles.map(x => x.min.lat).sort((a, b) => a - b); + const minLng = lngs[0]; + const minLat = lats[0]; + const maxLng = lngs[lngs.length - 1] + TILE_SIZE; + const maxLat = lats[lats.length - 1] + TILE_SIZE; + const boundingBox = BoundingBox(LngLat(minLng, minLat), LngLat(maxLng, maxLat)); + return boundingBox; + } +} diff --git a/server/api/services/osm.service.ts b/server/api/services/osm.service.ts index dcfbf521..6e12d7ed 100644 --- a/server/api/services/osm.service.ts +++ b/server/api/services/osm.service.ts @@ -7,8 +7,8 @@ import l from '../../common/logger'; import osm_fountain_config from '../../../config/fountains.sources.osm'; -import { FountainConfig } from '../../common/typealias'; -//TODO we could use overpas-ts thought I am not sure how well it is maintained and up-to-date +import { BoundingBox, FountainConfig } from '../../common/typealias'; +//TODO @ralf.hauser we could use overpas-ts thought I am not sure how well it is maintained and up-to-date import query_overpass from 'query-overpass'; interface OsmFountainConfigCollection { @@ -44,11 +44,9 @@ class OsmService { }); } - byBoundingBox(latMin: number, lngMin: number, latMax: number, lngMax: number): Promise { - // fetch fountain from OSM by coordinates + byBoundingBox(boundingBox: BoundingBox): Promise { return new Promise((resolve, reject) => { - const query = queryBuilderBox(latMin, lngMin, latMax, lngMax); - // l.info(query); + const query = queryBuilderBoundingBox(boundingBox); query_overpass( query, (error: any, data: OsmFountainConfigCollection) => { @@ -85,13 +83,16 @@ function queryBuilderCenter(lat: number, lng: number, radius = 10): string { `; } -function queryBuilderBox(latMin: number, lngMin: number, latMax: number, lngMax: number): string { +function queryBuilderBoundingBox(boundingBox: BoundingBox): string { // The querybuilder uses the sub_sources defined in osm_fountain_config to know which tags should be queried return ` (${['node', 'way'] .map(e => osm_fountain_config.sub_sources - .map(item => `${e}[${item.tag.name}=${item.tag.value}](${latMin},${lngMin},${latMax},${lngMax});`) + .map( + item => + `${e}[${item.tag.name}=${item.tag.value}](${boundingBox.min.lat},${boundingBox.min.lng},${boundingBox.max.lat},${boundingBox.max.lng});` + ) .join('') ) .join('')} diff --git a/server/api/services/processing.service.ts b/server/api/services/processing.service.ts index 2c4f7399..fe582f79 100644 --- a/server/api/services/processing.service.ts +++ b/server/api/services/processing.service.ts @@ -31,10 +31,10 @@ export function defaultCollectionEnhancement( // .then(r => fillOperatorInfo(r,dbg)) } -export function fillImageGalleries(fountainArr: Fountain[], city: string, debugAll: boolean): Promise { +export function fillImageGalleries(fountainArr: Fountain[], dbg: string, debugAll: boolean): Promise { // takes a collection of fountains and returns the same collection, // enhanced with image galleries when available or default images - l.info('processing.service.js starting fillImageGalleries: ' + city + ' debugAll ' + debugAll); + l.info('processing.service.js starting fillImageGalleries: ' + dbg + ' debugAll ' + debugAll); const promises: Promise[] = []; let i = 0; const tot = fountainArr.length; @@ -61,7 +61,7 @@ export function fillImageGalleries(fountainArr: Fountain[], city: string, debugA dbgAll = 0 == i % step; } const dbg = i + '/' + tot; - promises.push(WikimediaService.fillGallery(fountain, dbg, city, dbgAll, tot)); + promises.push(WikimediaService.fillGallery(fountain, dbg, dbgAll, tot)); }); return Promise.all(promises); } @@ -183,16 +183,7 @@ export function createUniqueIds(fountainArr: Fountain[]): Promise { export function essenceOf(fountainCollection: FountainCollection): FountainCollection { // returns a version of the fountain data with only the essential data - const newCollection: FountainCollection = { - type: 'FeatureCollection', - features: [], - }; - - //TODO @ralfhauser, properties do not exist on the GeoJSON standard for FeatureCollection, in other words, this is a hack. - (newCollection as any).properties = { - // Add last scan time info for https://github.com/water-fountains/proximap/issues/188 - last_scan: new Date(), - }; + const newCollection = FountainCollection([], fountainCollection.last_scan); // Get list of property names that are marked as essential in the metadata const essentialPropNames: string[] = _.map(fountain_property_metadata, (p, p_name) => { diff --git a/server/api/services/wikidata.service.ts b/server/api/services/wikidata.service.ts index a5c77362..744d462b 100644 --- a/server/api/services/wikidata.service.ts +++ b/server/api/services/wikidata.service.ts @@ -11,8 +11,10 @@ import { cacheAdapterEnhancer } from 'axios-extensions'; import * as _ from 'lodash'; import wdk from 'wikidata-sdk'; import sharedConstants from '../../common/shared-constants'; -import { Fountain } from '../../common/typealias'; +import { BoundingBox, Fountain } from '../../common/typealias'; import { MediaWikiEntityCollection, MediaWikiEntity, MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; +import { City } from '../../../config/locations'; +import { LNG_LAT_STRING_PRECISION } from './locationCache'; // Set up caching of http requests const http = axios.create({ @@ -27,7 +29,7 @@ const http = axios.create({ }); class WikidataService { - idsByCenter(lat: number, lng: number, radius = 10, locationName: string): Promise { + idsByCenter(lat: number, lng: number, radius = 10, city: City): Promise { // fetch fountain from OSM by coordinates, within radius in meters const sparql = ` SELECT ?place @@ -49,17 +51,15 @@ class WikidataService { bd:serviceParam wikibase:language "en,de,fr,it,tr" . } }`; - const res = doSparqlRequest(sparql, locationName, 'idsByCenter'); + const res = doSparqlRequest(sparql, 'idsByCenter for city ' + city); return res; } - idsByBoundingBox( - latMin: number, - lngMin: number, - latMax: number, - lngMax: number, - locationName: string - ): Promise { + idsByBoundingBox(bounds: BoundingBox): Promise { + const minLng = bounds.min.lng.toFixed(LNG_LAT_STRING_PRECISION); + const minLat = bounds.min.lat.toFixed(LNG_LAT_STRING_PRECISION); + const maxLng = bounds.max.lng.toFixed(LNG_LAT_STRING_PRECISION); + const maxLat = bounds.max.lat.toFixed(LNG_LAT_STRING_PRECISION); const sparql = ` SELECT ?place WHERE @@ -67,8 +67,8 @@ class WikidataService { SERVICE wikibase:box { # this service allows points within a box to be queried (https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_around_and_box) ?place wdt:P625 ?location . - bd:serviceParam wikibase:cornerWest "Point(${lngMin} ${latMin})"^^geo:wktLiteral. - bd:serviceParam wikibase:cornerEast "Point(${lngMax} ${latMax})"^^geo:wktLiteral. + bd:serviceParam wikibase:cornerSouthWest "Point(${minLng} ${minLat})"^^geo:wktLiteral. + bd:serviceParam wikibase:cornerNorthEast "Point(${maxLng} ${maxLat})"^^geo:wktLiteral. } . # The results of the spatial query are limited to instances or subclasses of water well (Q43483) or fountain (Q483453) @@ -76,14 +76,14 @@ class WikidataService { # the wikibase:label service allows the label to be returned easily. The list of languages provided are fallbacks: if no English label is available, use German etc. SERVICE wikibase:label { - bd:serviceParam wikibase:language "en,de,fr,it,tr" . + bd:serviceParam wikibase:language "${sharedConstants.LANGS.join(',')}" . } }`; - const res = doSparqlRequest(sparql, locationName, 'idsByBoundingBox'); + const res = doSparqlRequest(sparql, 'idsByBoundingBox for ' + JSON.stringify(bounds)); return res; } - byIds(qids: string[], locationName: string): Promise { + byIds(qids: string[], dbg: string): Promise { // fetch fountains by their QIDs const chunkSize = 50; // how many fountains should be fetched at a time (so as to not overload the server) return new Promise((resolve, reject) => { @@ -93,7 +93,7 @@ class WikidataService { chunk(qids, chunkSize).forEach(qidChunk => { chunkCount++; if (chunkSize * chunkCount > qids.length) { - l.info('wikidata.service.js byIds: chunk ' + chunkCount + ' for ' + locationName); + l.info('wikidata.service.js byIds: chunk ' + chunkCount + ' for ' + dbg); } // create sparql url const url = wdk.getEntities({ @@ -108,19 +108,13 @@ class WikidataService { Promise.all(httpPromises) .then(responses => { l.info( - 'wikidata.service.js byIds: ' + - chunkCount + - ' chunks of ' + - chunkSize + - ' prepared for loc "' + - locationName + - '"' + 'wikidata.service.js byIds: ' + chunkCount + ' chunks of ' + chunkSize + ' prepared for loc "' + dbg + '"' ); // holder for data of all fountains let dataAll: MediaWikiSimplifiedEntity[] = []; responses.forEach(r => { // holder for data from each chunk - //TODO should be typed as soon as we update wikidata-sdk to the latest version + //TODO @ralf.hauser should be typed as soon as we update wikidata-sdk to the latest version const data: MediaWikiSimplifiedEntity[] = []; for (const key in r.data.entities) { // simplify object structure of each wikidata entity and add it to 'data' @@ -138,19 +132,19 @@ class WikidataService { if (null != dataAll) { dataAllSize = dataAll.length; } - l.info('wikidata.service.js byIds: dataAll ' + dataAllSize + ' for loc "' + locationName + '"'); + l.info('wikidata.service.js byIds: dataAll ' + dataAllSize + ' for loc "' + dbg + '"'); } // return dataAll to //TODO @ralfhauser that's a smell, we should not use the resolve of an outer promise resolve(dataAll); }) .catch(e => { - l.error('wikidata.service.js byIds: catch e ' + e.stack + ' for loc "' + locationName + '"'); + l.error('wikidata.service.js byIds: catch e ' + e.stack + ' for loc "' + dbg + '"'); //TODO that's a smell, we should not use the reject of an outer promise reject(e); }); } catch (error: any) { - l.error('wikidata.service.js byIds: catch error ' + error.stack + ' for loc "' + locationName + '"'); + l.error('wikidata.service.js byIds: catch error ' + error.stack + ' for loc "' + dbg + '"'); reject(error); } @@ -198,7 +192,6 @@ class WikidataService { const latMin = undefined; const lngMax = undefined; const latMax = undefined; - const locationName = 'undefined'; const newQueryMiro = false; if (newQueryMiro) { @@ -221,7 +214,7 @@ class WikidataService { bd:serviceParam wikibase:language "en,de,fr,it,tr" . } }`; - const res = doSparqlRequest(sparql, locationName, 'fillArtistName'); + const res = doSparqlRequest(sparql, 'fillArtistName'); l.info('wikidata.service.js fillArtistName: new Miro response ' + res + ' "' + idWd + '"'); } @@ -507,7 +500,7 @@ function chunk(arr: T[], len: number): T[][] { return chunks; } -function doSparqlRequest(sparql: string, location: string, dbg: string): Promise { +function doSparqlRequest(sparql: string, dbg: string): Promise { return new Promise((resolve, reject) => { // create url from SPARQL const url = wdk.sparqlQuery(sparql); @@ -521,7 +514,7 @@ function doSparqlRequest(sparql: string, location: string, dbg: string): Promise const error = new Error( `wikidata.service.ts doSparqlRequest Request to Wikidata Failed. Status Code: ${res.status}. Status Message: ${res.statusText}. Url: ${url}` ); - l.error('wikidata.service.js doSparqlRequest: ' + dbg + ', location ' + location + ' ' + error.message); + l.error('wikidata.service.js doSparqlRequest: ' + dbg + ' ' + error.message); // consume response data to free up memory // TODO @ralfhauser, resume does not exist // res.resume(); @@ -533,33 +526,21 @@ function doSparqlRequest(sparql: string, location: string, dbg: string): Promise l.info( 'wikidata.service.js doSparqlRequest: ' + dbg + - ', location ' + - location + ' ' + //+simplifiedResults+' ' simplifiedResults.length + - ' ids found for ' + - location + ' ids found' ); resolve(simplifiedResults); } catch (e: any) { l.error( - 'wikidata.service.js doSparqlRequest: Error occurred simplifying wikidata results.' + - e.stack + - ' ' + - dbg + - ', location ' + - location + 'wikidata.service.js doSparqlRequest: Error occurred simplifying wikidata results.' + e.stack + ' ' + dbg ); reject(e); } }) .catch(error => { l.error( - `'wikidata.service.js doSparqlRequest: Request to Wikidata Failed. Url: ${url}` + - ' ' + - dbg + - ', location ' + - location + `'wikidata.service.js doSparqlRequest: Request to Wikidata Failed. Url: ${url}` + ' ' + dbg + '\n' + error ); reject(error); }); diff --git a/server/api/services/wikimedia.service.ts b/server/api/services/wikimedia.service.ts index ca179015..ff49d03d 100644 --- a/server/api/services/wikimedia.service.ts +++ b/server/api/services/wikimedia.service.ts @@ -46,7 +46,6 @@ class WikimediaService { private getImgsFromCats( fProps: FountainPropertyCollection, dbg: string, - city: string, dbgIdWd: string, name: string, imgNoInfoPomises: Promise[], @@ -63,8 +62,6 @@ class WikimediaService { ' commons categories defined "' + dbg + ' ' + - city + - ' ' + dbgIdWd + ' "' + name + @@ -86,8 +83,6 @@ class WikimediaService { '" "' + dbg + ' ' + - city + - ' ' + dbgIdWd + ' "' + name + @@ -102,8 +97,6 @@ class WikimediaService { ' commons categories defined "' + dbg + ' ' + - city + - ' ' + dbgIdWd + ' "' + name + @@ -111,19 +104,13 @@ class WikimediaService { ); } // lastCatName = catName; - const imgNoInfoPomise = getImgsOfCat(cat, dbg, city, imgUrlSet, imgUrls, dbgIdWd, fProps, debugAll); + const imgNoInfoPomise = getImgsOfCat(cat, dbg, imgUrlSet, imgUrls, dbgIdWd, fProps, debugAll); //TODO we might prioritize categories with small number of images to have greater variety of images? imgNoInfoPomises.push(imgNoInfoPomise); } } - fillGallery( - fountain: Fountain, - dbg: string, - city: string, - debugAll: boolean, - numbOfFntsInCollection: number - ): Promise { + fillGallery(fountain: Fountain, dbg: string, debugAll: boolean, numbOfFntsInCollection: number): Promise { //TODO @ralfhauser, changed it from null to '' to satisfy the type string let dbgIdWd = ''; // if (debugAll) { @@ -180,20 +167,18 @@ class WikimediaService { ' featured img) "' + dbg + ' ' + - city + - ' ' + dbgIdWd ); } //TODO @ralfhauser, changed it from boolean to empty array to satisfy types, In the end the array of imgNoInfoPomises is ignored anways, so I guess it does not matter imgNoInfoPomises.push(new Promise(resolve => resolve([]))); } else { - this.getImgsFromCats(fProps, dbg, city, dbgIdWd, name, imgNoInfoPomises, imgUrlSet, imgUrls, debugAll); + this.getImgsFromCats(fProps, dbg, dbgIdWd, name, imgNoInfoPomises, imgUrlSet, imgUrls, debugAll); } } else { // if not, resolve with empty if (process.env.NODE_ENV !== 'production' && debugAll) { - l.info('wikimedia.service.js: no commons category defined "' + dbg + ' ' + city + ' ' + dbgIdWd); + l.info('wikimedia.service.js: no commons category defined "' + dbg + ' ' + dbgIdWd); } //TODO @ralfhauser, changed it from boolean to empty array to satisfy types, In the end the array of imgNoInfoPomises is ignored anways, so I guess it does not matter imgNoInfoPomises.push(new Promise(resolve => resolve([]))); @@ -204,16 +189,7 @@ class WikimediaService { const totImgFound = imgUrlSet.size; if (0 < totImgFound) { if (debugAll) { - l.info( - 'wikimedia.service.js: fillGallery imgUrlSet.size ' + - totImgFound + - ' "' + - dbg + - ' ' + - city + - ' ' + - dbgIdWd - ); + l.info('wikimedia.service.js: fillGallery imgUrlSet.size ' + totImgFound + ' "' + dbg + ' ' + dbgIdWd); } if (MAX_IMG_SHOWN_IN_GALLERY < totImgFound) { l.info( @@ -223,8 +199,6 @@ class WikimediaService { totImgFound + ' "' + dbg + - '" ' + - city + ' ' + dbgIdWd ); @@ -252,8 +226,6 @@ class WikimediaService { '" already in other fountain(s) "' + dbg + ' ' + - city + - ' ' + dbgIdWd ); for (const clr of callers) { @@ -268,7 +240,7 @@ class WikimediaService { galValPromises.push( //TODO @ralfhauser getImageInfo returns Promise this statement most likely does not make sense // My guess, galValPromises should have nImg instead, hence I added the `then` after the catch, please check if this fix is correct - getImageInfo(nImg, k + '/' + imgL + ' "' + dbg + '" ' + city + ' ' + dbgIdWd, showDetails, fProps) + getImageInfo(nImg, k + '/' + imgL + ' "' + dbg + '" ' + dbgIdWd, showDetails, fProps) .catch(giiErr => { //TODO @ralfhauser, img.val does not exist, changed it to img.value, please check if this is correct l.info( @@ -277,8 +249,6 @@ class WikimediaService { '" ' + dbg + ' ' + - city + - ' ' + dbgIdWd + ' cat "' + img.cat + @@ -307,22 +277,13 @@ class WikimediaService { ' ' + dbg + ' ' + - city + - ' ' + dbgIdWd ); } return Promise.all(galValPromises).then(r => { if (debugAll) { l.info( - 'wikimedia.service.js: fillGallery galValPromises.r.length ' + - r.length + - ' ' + - dbg + - ' ' + - city + - ' ' + - dbgIdWd + 'wikimedia.service.js: fillGallery galValPromises.r.length ' + r.length + ' ' + dbg + ' ' + dbgIdWd ); } const allGalVal = galVal.concat(r); @@ -357,7 +318,7 @@ class WikimediaService { } else { //could check the qualifiers as per https://github.com/water-fountains/proximap/issues/294 if (debugAll) { - l.info('wikimedia.service.js: fillGallery ' + dbgIdWd + ' has no img ' + city + ' ' + dbg); + l.info('wikimedia.service.js: fillGallery ' + dbgIdWd + ' has no img ' + dbg); } return fountain; } @@ -728,7 +689,6 @@ export function getImageInfo( export function getImgsOfCat( cat: Category, dbg: string, - city: string, imgUrlSet: Set, imgUrls: ImageLike[], dbgIdWd: string, @@ -763,8 +723,6 @@ export function getImgsOfCat( ') images ' + dbg + ' ' + - city + - ' ' + dbgIdWd ); } @@ -812,8 +770,6 @@ export function getImgsOfCat( ') images ' + dbg + ' ' + - city + - ' ' + dbgIdWd ); l.info(rdErr['*']); diff --git a/server/common/ArrayExtensions.ts b/server/common/ArrayExtensions.ts new file mode 100644 index 00000000..604cb709 --- /dev/null +++ b/server/common/ArrayExtensions.ts @@ -0,0 +1,131 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ + +import { take } from 'lodash'; +import { compareI18n } from './compare'; +import { illegalState } from './illegalState'; + +export {}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +declare global { + interface Array { + /** + * @author Tegonal GmbH + * @license AGPL + */ + searchUnique(this: T[], predicate: (value: T) => boolean): T; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + isEmpty(this: T[]): boolean; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + nonEmpty(this: T[]): boolean; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + take(number: number): T[]; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + sortI18n(this: T[], propertyAccessFn?: (x: T, y: T) => [string, string]): T[]; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + groupBy(this: T[], keyGetter: (input: T) => K): Map; + } +} + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.searchUnique = function (this: T[], predicate: (value: T) => boolean): T { + const result = this.find(predicate); + if (result !== undefined) { + return result; + } else { + illegalState('searched in array for unique match, nothing found', this, 'used predicate', predicate); + } +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.isEmpty = function (this: T[]): boolean { + return this.length === 0; +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.nonEmpty = function (this: T[]): boolean { + return !this.isEmpty(); +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.take = function (this: T[], number: number): T[] { + return take(this, number); +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.sortI18n = function (this: T[], propertyAccessFn?: (a: T, b: T) => [string, string]): T[] { + return this.sort((a, b) => { + const [as, bs] = propertyAccessFn ? propertyAccessFn(a, b) : ['' + a, '' + b]; + + return compareI18n(as, bs); + }); +}; + +/** + * @description + * Takes an Array, and a grouping function, + * and returns a Map of the array grouped by the grouping function. + * + * @param this The receiver array of type V. + * @param keyGetter A Function that takes the the Array type V as an input, and returns a value of type K. + * K is generally intended to be a property key of V. + * + * @returns Map of the array grouped by the grouping function. + * + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.groupBy = function (this: V[], keyGetter: (input: V) => K): Map { + const map = new Map(); + this.forEach((item: V) => { + const key = keyGetter(item); + const collection = map.get(key); + if (!collection) { + map.set(key, [item]); + } else { + collection.push(item); + } + }); + return map; +}; diff --git a/server/common/build.info.ts b/server/common/build.info.ts index b07f204a..c94fb06f 100644 --- a/server/common/build.info.ts +++ b/server/common/build.info.ts @@ -1,9 +1,9 @@ // this file is automatically generated by git.version.js script const buildInfo = { version: '', - revision: '7acb2bc', - branch: '#150-forceRefresh-id', - commit_time: '2021-12-20 10:31:35 +0100', - build_time: 'Mon Dec 20 2021 11:11:32 GMT+0100 (Central European Standard Time)', + revision: 'aa6848b', + branch: '#148-load-by-bound', + commit_time: '2021-12-22 16:49:55 +0100', + build_time: 'Wed Dec 22 2021 17:13:05 GMT+0100 (Central European Standard Time)', }; export default buildInfo; diff --git a/server/common/compare.ts b/server/common/compare.ts new file mode 100644 index 00000000..4e2e7287 --- /dev/null +++ b/server/common/compare.ts @@ -0,0 +1,21 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ +export function compareI18n(a: string, b: string): number { + return new Intl.Collator([], { numeric: true }).compare(a, b); +} + +/** + * @author Tegonal GmbH + * @license AGPL + */ +export function compareBoolean(a: boolean, b: boolean, whenEqual: () => number): number { + if (a && !b) { + return -1; + } else if (!a && b) { + return 1; + } else { + return whenEqual(); + } +} diff --git a/server/common/illegalState.ts b/server/common/illegalState.ts new file mode 100644 index 00000000..d693ca06 --- /dev/null +++ b/server/common/illegalState.ts @@ -0,0 +1,13 @@ +import l from '../common/logger'; + +/** + * Since throw is not an expression in typescript you cannot do thing like ... || throw + * Hence this function + * @author Tegonal GmbH + * @license AGPL + */ +export function illegalState(msg: string, ...optionalParams: any[]): never { + const errorMsg = msg + ' ' + optionalParams.map(x => JSON.stringify(x)).join(' // '); + l.error(errorMsg); + throw new Error('IllegalState detected: ' + errorMsg); +} diff --git a/server/common/importAllExtensions.ts b/server/common/importAllExtensions.ts new file mode 100644 index 00000000..c84b4ed8 --- /dev/null +++ b/server/common/importAllExtensions.ts @@ -0,0 +1,5 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ +import './ArrayExtensions'; diff --git a/server/common/shared-constants.ts b/server/common/shared-constants.ts index a88c9ec7..38a71573 100644 --- a/server/common/shared-constants.ts +++ b/server/common/shared-constants.ts @@ -14,6 +14,7 @@ export default { //TODO proximap#394 reactivate serbian again, see also TODO in proximap LANGS: ['en', 'de', 'fr', 'it', 'tr' /* 'sr' */], - CACHE_FOR_HRS_i45db: 48, + CITY_TTL_IN_HOURS: 2 * 24, + BOUNDING_BOX_TTL_IN_HOURS: 7 * 24, gak: `${process.env.GOOGLE_API_KEY}`, }; diff --git a/server/common/sleep.ts b/server/common/sleep.ts new file mode 100644 index 00000000..23648f73 --- /dev/null +++ b/server/common/sleep.ts @@ -0,0 +1,8 @@ +/** + * primitive sleep function using setTimeout + * @author Tegonal GmbH + * @license AGPL + */ +export async function sleep(milliseconds: number): Promise { + return new Promise((resolve, _) => setTimeout(resolve, milliseconds)); +} diff --git a/server/common/swagger/Api.yaml b/server/common/swagger/Api.yaml index 20b6f6f9..2989ec3a 100644 --- a/server/common/swagger/Api.yaml +++ b/server/common/swagger/Api.yaml @@ -68,12 +68,12 @@ paths: example: wikidata required: true description: database for which the provided identifier is valid - - name: city + - name: loc in: query type: string - example: ch-zh + example: 47.3646083,8.5380421 required: true - description: code of city for which fountains are to be served + description: lat,lng of fountain, necessary in order that we can load - name: idval in: query type: string @@ -91,12 +91,20 @@ paths: get: description: Fetch fountains within bounding box parameters: - - name: city + - name: sw in: query type: string - example: ch-zh + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.3229261255644,8.45960259979614 required: true - description: code of location for which fountains are to be served + description: lat,lng of the south west location of the bounding box + - name: ne + in: query + type: string + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.431119712250506,8.61940272745742 + required: true + description: lat,lng of the north east location of the bounding box - name: refresh in: query type: boolean @@ -118,12 +126,20 @@ paths: get: description: Fetch list of processing errors for given location parameters: - - name: city + - name: sw + in: query + type: string + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.3229261255644,8.45960259979614 + required: true + description: lat,lng of the south west location of the bounding box + - name: ne in: query type: string - example: ch-zh + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.431119712250506,8.61940272745742 required: true - description: name of location for which fountain processing errors are to be served + description: lat,lng of the north east location of the bounding box responses: 200: description: Returns a collection of processing errors. diff --git a/server/common/swagger/index.ts b/server/common/swagger/index.ts index 5d4cc5b7..2f007feb 100644 --- a/server/common/swagger/index.ts +++ b/server/common/swagger/index.ts @@ -32,7 +32,6 @@ export function swaggerify(app: Express, routerProvider: (app: Express) => Route cookie: { secret: process.env.SESSION_SECRET, }, - // Don't allow JSON content over 100kb (default is 1mb) json: { limit: process.env.REQUEST_LIMIT, }, diff --git a/server/common/typealias.ts b/server/common/typealias.ts index 0258b79a..b4ec97a4 100644 --- a/server/common/typealias.ts +++ b/server/common/typealias.ts @@ -1,6 +1,9 @@ -import { Feature, FeatureCollection, Geometry, Point } from 'geojson'; +import { Feature, FeatureCollection, Geometry, Point, Position } from 'geojson'; +import { UncheckedBoundingBox } from '../../config/locations'; import { ImageLikeCollection, ImageLikeType } from '../../config/text2img'; +import { isNumeric } from '../api/controllers/utils'; import { PropStatus } from './constants'; +import { illegalState } from './illegalState'; import { Category, MediaWikiSimplifiedEntity } from './wikimedia-types'; // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap @@ -32,7 +35,11 @@ export type Fountain = FeatureCollection< G, FountainPropertyCollection> ->; +> & { last_scan: Date }; + +export function FountainCollection(fountains: Fountain[], lastScan: Date): FountainCollection { + return { type: 'FeatureCollection', features: fountains, last_scan: lastScan }; +} export type FountainConfig = SourceConfig; @@ -136,3 +143,53 @@ export type Database = SourceType; export function isDatabase(d: string): d is Database { return d === 'osm' || d === 'wikidata'; } + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export interface LngLat { + lng: number; + lat: number; +} +export function LngLat(lng: number, lat: number): LngLat { + if (lng < -180 || lng > 180) illegalState('lng out of range [-180, 180]', lng); + if (lat < -90 || lat > 90) illegalState('lat out of range [-180, 180]', lat); + + return { lng: lng, lat: lat }; +} +export function parseLngLat(lngLatAsString: string): LngLat { + const lngLatArr = lngLatAsString.split(','); + if (lngLatArr.length >= 2 && isNumeric(lngLatArr[0]) && isNumeric(lngLatArr[1])) { + const lat = Number(lngLatArr[0]); + const lng = Number(lngLatArr[1]); + return LngLat(lng, lat); + } else { + illegalState('could not parse to LngLat, given string: ' + lngLatAsString); + } +} +export function positionToLngLat(position: Position): LngLat { + const lng = position[0]; + const lat = position[1]; + if (lng === undefined || lat === undefined) { + illegalState('position.length was less than 2', position); + } + return LngLat(lng, lat); +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export interface BoundingBox { + min: LngLat; + max: LngLat; +} +export function BoundingBox(min: LngLat, max: LngLat): BoundingBox { + if (min.lng >= max.lng) illegalState('min lng greater or equal max lng.', 'min', min, 'max', max); + if (min.lat >= max.lat) illegalState('min lat greater or equal to max lat', 'min', min, 'max', max); + + return { min: min, max: max }; +} +export function uncheckedBoundingBoxToChecked(uncheckedBoundingBox: UncheckedBoundingBox): BoundingBox { + return BoundingBox( + LngLat(uncheckedBoundingBox.lngMin, uncheckedBoundingBox.latMin), + LngLat(uncheckedBoundingBox.lngMax, uncheckedBoundingBox.latMax) + ); +}