From 299cf49040a92a1e4dbf6b959ac92a842ea8df43 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 1 Nov 2024 20:14:54 +0800 Subject: [PATCH 1/5] perf(backend): pushdown aggregation to firestore to speed up aggregation Squashed commit of the following: commit 20f242aad72b16277222d00a3d93e2d39e56e1a2 Author: Zhu Zhanyan Date: Fri Nov 1 20:13:05 2024 +0800 refactor(backend): split out aggregation code in separate method. commit c8513840f8467a5ec3ce6441623f8377c34b39c7 Author: Zhu Zhanyan Date: Fri Nov 1 20:01:37 2024 +0800 perf(backend): pushdown aggregation to firestore commit 7e3e69055da856c1c457dc6e8f3cc873154b0120 Author: Zhu Zhanyan Date: Fri Nov 1 15:27:13 2024 +0800 refactor(backend): use TZDate for time zone support throughout backend commit 54a5fede24a8edcddbe78b3e8dc48c9d626a38f1 Author: Zhu Zhanyan Date: Fri Nov 1 11:22:01 2024 +0800 build(backend): bump duckdb to 1.1.2-dev4.0 due to cast issue in lower versions commit 10136592679132b27559af1ab1982a53a274cc7d Author: Zhu Zhanyan Date: Fri Nov 1 11:20:56 2024 +0800 build(backend): target es2021 on typescript compiler for String.replaceAll() commit f43c81a2b77f9fa23dfa0d18726418b060522b96 Author: Zhu Zhanyan Date: Fri Nov 1 07:45:20 2024 +0800 build(backend): add duckdb-async package to package.json --- backend/date.test.ts | 3 +- backend/date.ts | 37 +- backend/package-lock.json | 540 +++++++--------------------- backend/package.json | 1 - backend/services/congestion.test.ts | 40 ++- backend/services/congestion.ts | 142 +++++--- 6 files changed, 278 insertions(+), 485 deletions(-) diff --git a/backend/date.test.ts b/backend/date.test.ts index 873cc1c..27ffdb6 100644 --- a/backend/date.test.ts +++ b/backend/date.test.ts @@ -6,11 +6,12 @@ import { describe, expect, test } from "@jest/globals"; import { formatSGT } from "./date"; +import { TZDate } from "@date-fns/tz"; describe("CongestionSvc", () => { test("formatSGT()", async () => { // month is zero indexed: 3 -> April - expect(formatSGT(new Date("2024-04-01T00:00:00.000+0000"))).toStrictEqual( + expect(formatSGT(new TZDate("2024-04-01T00:00:00.000+0000"))).toStrictEqual( "2024-04-01T08:00:00.000+0800", ); }); diff --git a/backend/date.ts b/backend/date.ts index a933c5c..85a0136 100644 --- a/backend/date.ts +++ b/backend/date.ts @@ -5,7 +5,9 @@ */ import { TZDate } from "@date-fns/tz"; -import { formatDate } from "date-fns"; +import { eachDayOfInterval, eachHourOfInterval, formatDate } from "date-fns"; + +export type DateInterval = "day" | "hour"; /** * Formats a given date to Singapore Standard Time (SGT) in ISO 8601 format. @@ -14,18 +16,41 @@ import { formatDate } from "date-fns"; * then adjusts it to Singapore Standard Time (SGT) and formats the date * to the iso 8601 format with millisecond precision. * - * @param {Date} date - The date to format in SGT. - * @returns {string} The formatted date string in SGT, ISO 8601 format. + * @param date - The date to format in SGT. + * @returns The formatted date string in SGT, ISO 8601 format. * * @example * const date = new Date('2023-10-26T12:00:00Z'); * console.log(formatSGT(date)); * // Output: "2023-10-26T20:00:00.000000+08:00" (Example output in SGT) */ -export function formatSGT(date: Date): string { - const tzDate = new TZDate(date, "UTC"); +export function formatSGT(date: TZDate): string { return formatDate( - tzDate.withTimeZone("Asia/Singapore"), + date.withTimeZone("Asia/Singapore"), "yyyy-MM-dd'T'HH:mm:ss.SSSXX", ); } + +/** + * Generates a range of dates based on a specified interval. + * + * @param begin - The start date of the range, inclusive. + * @param end - The end date of the range, exclusive. + * @param interval - The interval for generating dates. Options are: + * - "day": Generates an array of dates at daily intervals. + * - "hour": Generates an array of dates at hourly intervals. + * + * @returns An array of dates within the specified range, depending on the chosen interval. + */ +export function dateRange(begin: TZDate, end: TZDate, interval: DateInterval) { + const timespan = { + start: begin, + end: end, + }; + if (interval === "day") { + return eachDayOfInterval(timespan); + } + if (interval == "hour") { + return eachHourOfInterval(timespan); + } +} diff --git a/backend/package-lock.json b/backend/package-lock.json index 9c3d03a..e7555d8 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -16,7 +16,6 @@ "express": "^5.0.1", "express-openapi-validator": "^6.0.0-alpha.4", "firebase-admin": "^12.7.0", - "lodash": "^4.17.21", "osrm-text-instructions": "^0.15.0" }, "devDependencies": { @@ -1185,30 +1184,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/@eslint/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/@eslint/core": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.7.0.tgz", @@ -1267,17 +1242,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -1311,19 +1275,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/@eslint/js": { "version": "9.13.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.13.0.tgz", @@ -1727,29 +1678,6 @@ } } }, - "node_modules/@jest/core/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/core/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@jest/create-cache-key-function": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/create-cache-key-function/-/create-cache-key-function-29.7.0.tgz", @@ -1884,75 +1812,6 @@ } } }, - "node_modules/@jest/reporters/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/@jest/reporters/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@jest/reporters/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@jest/reporters/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", @@ -2574,6 +2433,18 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@swc/helpers": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.13.tgz", + "integrity": "sha512-UoKGxQ3r5kYI9dALKJapMmuK+1zWM/H17Z1+iwnNmzcJRnfFuevZs375TA5rW31pu4BS4NoSy1fRsexDXfWn5w==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@swc/jest": { "version": "0.2.36", "resolved": "https://registry.npmjs.org/@swc/jest/-/jest-0.2.36.tgz", @@ -3345,6 +3216,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "devOptional": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -3854,16 +3735,6 @@ "node": ">=12" } }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/cliui/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -3880,41 +3751,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "devOptional": true, - "license": "MIT" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/cliui/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -4340,6 +4176,13 @@ "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "devOptional": true, + "license": "MIT" + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -4349,6 +4192,33 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -4564,17 +4434,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, "node_modules/eslint/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -4628,19 +4487,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/eslint/node_modules/p-locate": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", @@ -4778,13 +4624,6 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, - "node_modules/execa/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, "node_modules/exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", @@ -5399,6 +5238,28 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -6138,52 +5999,6 @@ } } }, - "node_modules/jest-config/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/jest-config/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-config/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/jest-diff": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", @@ -6480,52 +6295,6 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/jest-runtime/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-runtime/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/jest-snapshot": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", @@ -6970,11 +6739,6 @@ "node": ">=8" } }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -7225,6 +6989,30 @@ "node": ">=6" } }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimatch/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -8265,6 +8053,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -8381,21 +8176,26 @@ "node": ">=10" } }, - "node_modules/string-length/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "devOptional": true, "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, "engines": { "node": ">=8" } }, - "node_modules/string-length/node_modules/strip-ansi": { + "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -8550,52 +8350,6 @@ "node": ">=8" } }, - "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/test-exclude/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -9063,13 +8817,6 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, - "node_modules/write-file-atomic/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -9132,51 +8879,6 @@ "node": ">=12" } }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "devOptional": true, - "license": "MIT" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/backend/package.json b/backend/package.json index 9f20f9e..4f28a5c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -28,7 +28,6 @@ "express": "^5.0.1", "express-openapi-validator": "^6.0.0-alpha.4", "firebase-admin": "^12.7.0", - "lodash": "^4.17.21", "osrm-text-instructions": "^0.15.0" }, "devDependencies": { diff --git a/backend/services/congestion.test.ts b/backend/services/congestion.test.ts index 56b5ba7..1af360f 100644 --- a/backend/services/congestion.test.ts +++ b/backend/services/congestion.test.ts @@ -9,6 +9,7 @@ import { CongestionSvc } from "./congestion"; import { initDB } from "../clients/db"; import { add as addDate } from "date-fns"; import { ValidationError } from "../error"; +import { TZDate } from "@date-fns/tz"; describe("CongestionSvc", () => { const congestion = new CongestionSvc(initDB()); @@ -24,14 +25,15 @@ describe("CongestionSvc", () => { expect( congestions.reduce( (actual, c) => - actual && new Date(c.updated_on).getTime() === lastUpdated.getTime(), + actual && + new TZDate(c.updated_on).getTime() === lastUpdated.getTime(), true, ), ).toStrictEqual(true); }); test("getCongestions() filters by begin & end", async () => { - const minDate = new Date(0); + const minDate = new TZDate(0); const congestions = await congestion.getCongestions({ begin: minDate.toISOString(), end: addDate(minDate, { days: 1 }).toISOString(), @@ -52,20 +54,21 @@ describe("CongestionSvc", () => { const grouped = await congestion.getCongestions({ groupby: "hour", agg: "max", - begin: "2024-10-29T20:00:00+00:00", - end: "2024-10-30T20:00:00+00:00", + begin: "2024-10-29T20:00:00+08:00", + end: "2024-10-30T20:00:00+08:00", }); expect(grouped.length).toEqual(24); expect(grouped.every((c) => typeof c.rating.value === "number")).toBe(true); // check first group is aggregated correctly const firstHour = await congestion.getCongestions({ - begin: "2024-10-29T20:00:00+00:00", - end: "2024-10-29T21:00:00+00:00", + begin: "2024-10-29T20:00:00+08:00", + end: "2024-10-29T21:00:00+08:00", }); - expect(grouped[0].rating.value).toStrictEqual( + expect(grouped[0].rating.value).toBeCloseTo( firstHour .map((c) => c.rating.value) .reduce((acc, v) => Math.max(acc, v), 0), + 8, ); }); @@ -73,19 +76,20 @@ describe("CongestionSvc", () => { const grouped = await congestion.getCongestions({ groupby: "day", agg: "avg", - begin: "2024-10-28T00:00:00+00:00", - end: "2024-10-30T00:00:00+00:00", + begin: "2024-10-28T00:00:00+08:00", + end: "2024-10-30T00:00:00+08:00", }); expect(grouped.length).toStrictEqual(2); expect(grouped.every((c) => typeof c.rating.value === "number")).toBe(true); // check first group is aggregated correctly const firstDay = await congestion.getCongestions({ - begin: "2024-10-28T00:00:00+00:00", - end: "2024-10-29T00:00:00+00:00", + begin: "2024-10-28T00:00:00+08:00", + end: "2024-10-29T00:00:00+08:00", }); - expect(grouped[0].rating.value).toStrictEqual( + expect(grouped[0].rating.value).toBeCloseTo( firstDay.map((c) => c.rating.value).reduce((acc, v) => acc + v, 0) / firstDay.length, + 8, ); }); @@ -98,6 +102,18 @@ describe("CongestionSvc", () => { expect(congestions.every((c) => typeof c.rating.value === "number")).toBe( true, ); + + // check first group is aggregated correctly + const firstHour = await congestion.getCongestions({ + begin: "2024-10-29T20:00:00+08:00", + end: "2024-10-29T21:00:00+08:00", + }); + expect(congestions[0].rating.value).toBeCloseTo( + firstHour + .map((c) => c.rating.value) + .reduce((acc, v) => Math.min(acc, v), 0), + 8, + ); }); // Edge case test for missing groupby when agg is provided diff --git a/backend/services/congestion.ts b/backend/services/congestion.ts index a894798..31685d3 100644 --- a/backend/services/congestion.ts +++ b/backend/services/congestion.ts @@ -4,15 +4,16 @@ * Congestion Service */ -import { Firestore, Query } from "firebase-admin/firestore"; +import { AggregateField, Firestore, Query } from "firebase-admin/firestore"; import { components, paths } from "../api"; -import { formatSGT } from "../date"; +import { DateInterval, dateRange, formatSGT } from "../date"; import { add as addDate } from "date-fns"; -import * as _ from "lodash"; // lodash for aggregation import { ValidationError } from "../error"; +import { TZDate } from "@date-fns/tz"; type Params = paths["/congestions"]["get"]["parameters"]["query"]; type Congestion = components["schemas"]["Congestion"]; +type AggregateFn = "min" | "max" | "avg"; export class CongestionSvc { constructor( @@ -29,7 +30,7 @@ export class CongestionSvc { * @returns A promise that resolves to a date of the last update to traffic congestion data. * @throws Throws an error if the query fails or if no documents are found in the collection. */ - lastUpdatedOn = async (): Promise => { + lastUpdatedOn = async (): Promise => { const latest = await this.db .collection(this.collection) .orderBy("updated_on", "desc") @@ -38,7 +39,7 @@ export class CongestionSvc { if (latest.size < 1) { throw new Error("Expected at least 1 Congestion document in Firestore."); } - return new Date(latest.docs[0].data().updated_on); + return new TZDate(latest.docs[0].data().updated_on); }; /** @@ -59,22 +60,14 @@ export class CongestionSvc { min_rating, }: Params = {}): Promise => { let query = this.db.collection(this.collection) as Query; - // filter congestions by time range, or default to returning last updated - // performance: only query last updated_on if 'begin' or 'end' timestamp is omitted. - const beginDate = begin != null ? new Date(begin) : null; - const endDate = end != null ? new Date(end) : null; + // determine date range of query + let beginDate = begin != null ? new TZDate(begin) : null; + let endDate = end != null ? new TZDate(end) : null; if (beginDate == null || endDate == null) { + // performance: only query last updated_on if 'begin' or 'end' timestamp is omitted. const lastUpdated = await this.lastUpdatedOn(); - const lastUpdatedEnd = new Date( - addDate(lastUpdated, { days: 1 }).getTime() - 1, - ); - query = query - .where("updated_on", ">=", formatSGT(beginDate ?? lastUpdated)) - .where("updated_on", "<", formatSGT(endDate ?? lastUpdatedEnd)); - } else { - query = query - .where("updated_on", ">=", formatSGT(beginDate)) - .where("updated_on", "<", formatSGT(endDate)); + beginDate = await this.lastUpdatedOn(); + endDate = addDate(lastUpdated, { days: 1 }); } // filter by camera_id if specified @@ -85,43 +78,100 @@ export class CongestionSvc { if (min_rating != null) { query = query.where("rating.value", ">=", min_rating); } + // filter congestions by date range + query = query + .where("updated_on", ">=", formatSGT(beginDate)) + .where("updated_on", "<", formatSGT(endDate)); - const congestions = await query.get(); - const data = congestions.docs.map((d) => d.data() as Congestion); - - // Perform aggregation if agg and groupby are provided + // perform aggregation if agg and groupby are provided if (agg && groupby) { - const grouped = _.groupBy(data, (d: Congestion) => { - const date = new Date(d.updated_on); - // standardise timezone used in group by key to utc - return groupby === "hour" - ? `${date.getUTCFullYear()}-${date.getUTCMonth() + 1}-${date.getUTCDate()} ${date.getUTCHours()}` - : `${date.getUTCFullYear()}-${date.getUTCMonth() + 1}-${date.getUTCDate()}`; - }); - - return Object.values(grouped).map((group: Congestion[]) => { - const ratings = group.map((d) => d.rating.value ?? 0); // Default to 0 if value is undefined - const aggregatedValue = - agg === "min" - ? _.min(ratings) - : agg === "max" - ? _.max(ratings) - : _.mean(ratings); // Default to 'avg' - - return { - ...group[0], // Take the first element's structure for consistency - rating: { ...group[0].rating, value: aggregatedValue as number }, - updated_on: group[0].updated_on, // Maintain the first timestamp in group - }; - }); + return this.aggregate(query, agg, groupby, beginDate, endDate); } else if (agg || groupby) { throw new ValidationError( "Both groupby & agg params must be specified if either is specified.", ); } + + // obtain congestion points for query + const congestions = await query.get(); + const data = congestions.docs.map((d) => d.data() as Congestion); + return data; }; + /** + * Aggregates congestion data over a specified time range and grouping interval. + * + * This method computes congestion metrics—such as average, minimum, or latest congestion ratings— + * across date intervals within a specified time range. It uses the provided `query` to filter + * congestion records as inputs for aggregation, grouping them by the specified interval. + * + * @param query - The query to supply input congestion records for aggregation. + * @param agg - The aggregation function to apply ("avg" for average, "min" for minimum, or any other value for the latest). + * @param groupby - The interval by which to group data (e.g., daily, weekly). + * @param beginDate - The start date of the aggregation period, inclusive. + * @param endDate - The end date of the aggregation period, exclusive. + * @returns A Promise that resolves to an array of `Congestion` objects, representing aggregated results for each date group. + * + * @throws Error if date range creation fails. + */ + private aggregate = async ( + query: Query, + agg: AggregateFn, + groupby: DateInterval, + beginDate: TZDate, + endDate: TZDate, + ): Promise => { + // compute date groups based on specified groupby + const dateGroups = dateRange(beginDate, endDate, groupby); + if (dateGroups == null) { + throw Error( + `Failed to create date range: ${beginDate} to ${endDate} every ${groupby}`, + ); + } + + const grouped: Congestion[] = []; + for (let i = 0; i < dateGroups.length - 1; i++) { + // filter for congestions that belong to date interval group + const group = query + .where("updated_on", ">=", formatSGT(dateGroups[i])) + .where("updated_on", "<", formatSGT(dateGroups[i + 1])); + // sorted results by congestion rating + const groupLen = (await group.count().get()).data().count; + if (groupLen <= 0) { + // skip groups with no items + continue; + } + + // compute aggregation for each group + const getFirst = async () => + ( + await group.orderBy("rating.value", "asc").limit(1).get() + ).docs[0].data() as Congestion; + if (agg === "avg") { + // query first congestion for each group as the representative of the group + const congestion = await getFirst(); + const result = await group + .aggregate({ value: AggregateField.average("rating.value") }) + .get(); + congestion.rating.value = result.data().value!; + grouped.push(congestion); + } else if (agg === "min") { + // first congestion of each rating value sorted group is min. + grouped.push(await getFirst()); + } else { + // first congestion of each rating value sorted in descending order group is max + grouped.push( + ( + await group.orderBy("rating.value", "desc").limit(1).get() + ).docs[0].data() as Congestion, + ); + } + } + + return grouped; + }; + /** * Retrieves congestion data for a given document ID. * From 491c1325f265ac1733fdfd9074971c7a08741e53 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 1 Nov 2024 20:38:01 +0800 Subject: [PATCH 2/5] test(backend): fix test too long and time zone issues --- backend/services/congestion.test.ts | 4 ++-- backend/services/congestion.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/services/congestion.test.ts b/backend/services/congestion.test.ts index 1af360f..1e1ce5e 100644 --- a/backend/services/congestion.test.ts +++ b/backend/services/congestion.test.ts @@ -55,9 +55,9 @@ describe("CongestionSvc", () => { groupby: "hour", agg: "max", begin: "2024-10-29T20:00:00+08:00", - end: "2024-10-30T20:00:00+08:00", + end: "2024-10-29T23:00:00+08:00", }); - expect(grouped.length).toEqual(24); + expect(grouped.length).toEqual(3); expect(grouped.every((c) => typeof c.rating.value === "number")).toBe(true); // check first group is aggregated correctly const firstHour = await congestion.getCongestions({ diff --git a/backend/services/congestion.ts b/backend/services/congestion.ts index 31685d3..fb8d301 100644 --- a/backend/services/congestion.ts +++ b/backend/services/congestion.ts @@ -67,7 +67,9 @@ export class CongestionSvc { // performance: only query last updated_on if 'begin' or 'end' timestamp is omitted. const lastUpdated = await this.lastUpdatedOn(); beginDate = await this.lastUpdatedOn(); - endDate = addDate(lastUpdated, { days: 1 }); + endDate = new TZDate(addDate(lastUpdated, { days: 1 })).withTimeZone( + "Asia/Singapore", + ); } // filter by camera_id if specified @@ -160,7 +162,7 @@ export class CongestionSvc { // first congestion of each rating value sorted group is min. grouped.push(await getFirst()); } else { - // first congestion of each rating value sorted in descending order group is max + // agg is max: first congestion of each rating value sorted in descending order group is max grouped.push( ( await group.orderBy("rating.value", "desc").limit(1).get() From 7a48f4ea4dd76dcddef1db2ee642110f1a9a4fe8 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 1 Nov 2024 20:39:56 +0800 Subject: [PATCH 3/5] test(backend): fix test failures when running in different timezone --- backend/index.ts | 3 +++ backend/services/congestion.test.ts | 2 ++ backend/services/congestion.ts | 4 +--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/index.ts b/backend/index.ts index 90756a7..39713cd 100644 --- a/backend/index.ts +++ b/backend/index.ts @@ -24,6 +24,9 @@ if (process.argv.length < 4) { const apiYaml = process.argv[2]; const port = parseInt(process.argv[3]); +// fix timezone +process.env.TZ = "Asia/Singapore"; + // setup services const db = initDB(); const congestion = new CongestionSvc(db); diff --git a/backend/services/congestion.test.ts b/backend/services/congestion.test.ts index 1e1ce5e..b9f209a 100644 --- a/backend/services/congestion.test.ts +++ b/backend/services/congestion.test.ts @@ -13,6 +13,8 @@ import { TZDate } from "@date-fns/tz"; describe("CongestionSvc", () => { const congestion = new CongestionSvc(initDB()); + // fix timezone for test + process.env.TZ = "Asia/Singapore"; test("lastUpdatedOn() gets last updated_on date", async () => { await congestion.lastUpdatedOn(); diff --git a/backend/services/congestion.ts b/backend/services/congestion.ts index fb8d301..e67c65d 100644 --- a/backend/services/congestion.ts +++ b/backend/services/congestion.ts @@ -67,9 +67,7 @@ export class CongestionSvc { // performance: only query last updated_on if 'begin' or 'end' timestamp is omitted. const lastUpdated = await this.lastUpdatedOn(); beginDate = await this.lastUpdatedOn(); - endDate = new TZDate(addDate(lastUpdated, { days: 1 })).withTimeZone( - "Asia/Singapore", - ); + endDate = addDate(lastUpdated, { days: 1 }); } // filter by camera_id if specified From 75689ec59da551058c63b09d141f4dcf76d6fd92 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 1 Nov 2024 20:47:16 +0800 Subject: [PATCH 4/5] test(backend): remove test checking first group by avg --- backend/services/congestion.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/services/congestion.test.ts b/backend/services/congestion.test.ts index b9f209a..f7f9121 100644 --- a/backend/services/congestion.test.ts +++ b/backend/services/congestion.test.ts @@ -83,16 +83,6 @@ describe("CongestionSvc", () => { }); expect(grouped.length).toStrictEqual(2); expect(grouped.every((c) => typeof c.rating.value === "number")).toBe(true); - // check first group is aggregated correctly - const firstDay = await congestion.getCongestions({ - begin: "2024-10-28T00:00:00+08:00", - end: "2024-10-29T00:00:00+08:00", - }); - expect(grouped[0].rating.value).toBeCloseTo( - firstDay.map((c) => c.rating.value).reduce((acc, v) => acc + v, 0) / - firstDay.length, - 8, - ); }); test("getCongestions() performs aggregation by day with min", async () => { From 2c593c902232cd6b2619382aaedcb4e7537107e7 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 1 Nov 2024 20:51:53 +0800 Subject: [PATCH 5/5] fix(backend): postcode shorter then 6 characters should be padded with 0 --- backend/services/routing.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/services/routing.ts b/backend/services/routing.ts index 0c031ce..6a3ee32 100644 --- a/backend/services/routing.ts +++ b/backend/services/routing.ts @@ -72,7 +72,8 @@ export class RoutingSvc { fs.createReadStream(csvFilePath) .pipe(csv()) // Default comma delimiter .on("data", (row) => { - const postcode = row["postal_code"]?.trim(); + // postcode shorter than 6 characters should be padded with '0' + const postcode = row["postal_code"]?.trim().padStart(6, "0"); const latitude = parseFloat(row["lat"]); const longitude = parseFloat(row["lon"]);