From eb61651225f5e89f37676af64f8f9e3b0440624e Mon Sep 17 00:00:00 2001 From: Seb-sti1 <65665540+seb-sti1@users.noreply.github.com> Date: Sat, 16 Dec 2023 15:40:29 +0100 Subject: [PATCH] Add Valhalla --- backend/src/upload/file.processor.ts | 26 +++-- backend/src/upload/upload.service.ts | 59 ++++++++--- backend/src/upload/valhalla.ts | 141 +++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 18 deletions(-) create mode 100644 backend/src/upload/valhalla.ts diff --git a/backend/src/upload/file.processor.ts b/backend/src/upload/file.processor.ts index d76067d3..78e51b1e 100644 --- a/backend/src/upload/file.processor.ts +++ b/backend/src/upload/file.processor.ts @@ -83,10 +83,18 @@ export class FileProcessor { */ @Process('process-file') async handleFileProcessing(job: Job<{ filePath: string }>) { + const printJobInfo = (...args: any[]) => { + console.log(`[${job.id} ${job.progress}%]`, ...args); + }; + + const printJobError = (...args: any[]) => { + console.error(`[${job.id} ${job.progress}%]`, ...args); + }; + try { const { filePath } = job.data; const debug = process.env.IMPORT_DEBUG === 'true'; - console.log(`Processing file: ${filePath} (job id: ${job.id})`); + printJobInfo(`Processing file: ${filePath}`); //here we make sure that there is at least one RSP file and a HDC directory let surveys = find_surveys(filePath, debug); @@ -95,9 +103,7 @@ export class FileProcessor { } if (surveys.length == 0) { - if (debug) { - console.log('No valid data found in directory: ' + filePath); - } + printJobInfo('No valid data found in directory: ' + filePath); } // TODO: split process here instead of for the all zip file (one process per survey) @@ -109,11 +115,19 @@ export class FileProcessor { ); const data = extract_measurements_data(surveys[i], debug); + if (!(await this.service.mapMatch(surveys[i], data))) { + printJobError('Failed to map match data.'); + } const roadImages = extract_road_image_data(surveys[i], debug); - const dashcameraImages = extract_dashcam_image_data(surveys[i], debug); + if (!(await this.service.mapMatch(surveys[i], roadImages))) { + printJobError('Failed to map match road images.'); + } - // TODO(Seb-sti1): (when rest working) add valhalla here + const dashcameraImages = extract_dashcam_image_data(surveys[i], debug); + if (!(await this.service.mapMatch(surveys[i], dashcameraImages))) { + printJobError('Failed to map match dashcam images.'); + } // Upload all data and images to the database await Promise.all([ diff --git a/backend/src/upload/upload.service.ts b/backend/src/upload/upload.service.ts index 6c4a30c8..a1071b15 100644 --- a/backend/src/upload/upload.service.ts +++ b/backend/src/upload/upload.service.ts @@ -1,6 +1,11 @@ import { Injectable } from '@nestjs/common'; import { InjectConnection, Knex } from 'nestjs-knex'; -import { SurveyImage, SurveyRoadParameters, SurveyStructure } from './upload'; +import { + SurveyData, + SurveyImage, + SurveyRoadParameters, + SurveyStructure, +} from './upload'; import { join } from 'path'; import { copyFileSync, existsSync, mkdirSync } from 'fs'; @@ -9,6 +14,7 @@ import * as sharp from 'sharp'; import { OSMWayId } from '../models'; import { IWay, Way } from '../tables'; import { getOSMWaysInARoad } from './osm'; +import { getGpsPointAtDistance, valhalla } from './valhalla'; @Injectable() export class UploadService { @@ -199,8 +205,10 @@ export class UploadService { /** * Get or create the ways corresponding to the OSMWayIds. - * You should definitely use this function instead of getOrCreateWay if you - * have multiple ways to get. + * + * You should definitely use this function instead of getOrCreateWay if you have multiple ways that + * are likely on the same road. + * * This executes the queries sequentially to avoid useless queries to OSM * and the database. This is especially useful when the ways are in the same road. * @@ -214,15 +222,10 @@ export class UploadService { ); return await tasks.reduce(async (cur: Promise, next: string) => { - return cur.then(async (value: IWay[]) => { - if (((value.length / tasks.length) * 100) % 10 === 0) - console.info( - 'Importation status', - `${(value.length / tasks.length) * 100}%`, - ); - - return [...value, await this.getOrCreateWay(next)]; - }); + return cur.then(async (value: IWay[]) => [ + ...value, + await this.getOrCreateWay(next), + ]); }); } @@ -281,4 +284,36 @@ export class UploadService { return way[0]; } + + async mapMatch(survey: SurveyStructure, data: SurveyData[]) { + let geometry = survey.geometry; + + let positions = data.map((d) => + getGpsPointAtDistance(geometry, d.distance_survey / 1000), + ); + + // use valhalla to map match the data + const matched = await valhalla(positions); + if (matched === false) { + return false; + } + + //TODO: if there are result for every point + + // get or create the ways (also inserting the road if needed) using osm + const way = await this.getOrCreateWays( + matched.map((m) => String(m.way_id)), + ); + if (way === false) { + return false; + } + + for (let i = 0; i < matched.length; i++) { + data[i].fk_way_id = way[i].id; + data[i].distance_way = matched[i].distance_way; + data[i].position = positions[i]; + } + + return true; + } } diff --git a/backend/src/upload/valhalla.ts b/backend/src/upload/valhalla.ts new file mode 100644 index 00000000..8596b264 --- /dev/null +++ b/backend/src/upload/valhalla.ts @@ -0,0 +1,141 @@ +import { LatLng } from '../models'; + +/** + * Get the coordinates of the point at the given distance along the path + * @param coordinates path from the .rsp file + * @param givenDistance distance in km + * @param debug + * @returns { lat: number; lon: number } the coordinates of the point at the given distance + * + * @author Liwei + */ +export function getGpsPointAtDistance( + coordinates: LatLng[], + givenDistance: number, + debug: boolean = false, +): LatLng { + function calculateDistance(coord1: LatLng, coord2: LatLng) { + const earthRadius = 6371; //Unit: kilometers + const [lat1, lon1] = [coord1.lat, coord1.lng]; + const [lat2, lon2] = [coord2.lat, coord2.lng]; + const dLat = (lat2 - lat1) * (Math.PI / 180); + const dLon = (lon2 - lon1) * (Math.PI / 180); + const a = + Math.sin(dLat / 2) ** 2 + + Math.cos(lat1 * (Math.PI / 180)) * + Math.cos(lat2 * (Math.PI / 180)) * + Math.sin(dLon / 2) ** 2; + const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); //Use the Haversine formula + return earthRadius * c; //// Returns the distance between two points in kilometers. + } + + let remainingDistance = givenDistance; + let targetSegmentIndex = -1; + + if (debug) + console.debug( + `before for, remainingDistance: ${remainingDistance}, targetSegmentIndex: ${targetSegmentIndex}`, + ); + + for (let i = 0; i < coordinates.length - 1; i++) { + const segmentDistance = calculateDistance( + coordinates[i], + coordinates[i + 1], + ); + if (remainingDistance > segmentDistance) { + remainingDistance -= segmentDistance; + } else { + targetSegmentIndex = i; + break; + } + } + + if (targetSegmentIndex === -1) { + console.warn( + 'The given distance is longer than the path length. The last point of the path is returned.', + ); + return coordinates[coordinates.length - 1]; + } + + if (debug) + console.debug( + `remainingDistance: ${remainingDistance}, targetSegmentIndex: ${targetSegmentIndex}`, + ); + + const [startLat, startLon] = [ + coordinates[targetSegmentIndex].lat, + coordinates[targetSegmentIndex].lng, + ]; + const [endLat, endLon] = [ + coordinates[targetSegmentIndex + 1].lat, + coordinates[targetSegmentIndex + 1].lng, + ]; + + if (debug) console.debug(startLat, startLon, endLat, endLon); + + const ratio = + remainingDistance / + calculateDistance( + coordinates[targetSegmentIndex], + coordinates[targetSegmentIndex + 1], + ); // Calculate the proportion of the target point's position in the current segment. + const targetLatitude = startLat + ratio * (endLat - startLat); + const targetLongitude = startLon + ratio * (endLon - startLon); + + return { lat: targetLatitude, lng: targetLongitude }; +} + +/** + * The path of data that needs to be map matched. + * @param data the path of data. + * @result for each point of the data array, the way_id, the distance along the way and the length of the way is returned. + * + * @author Kerbourc'h + */ +export async function valhalla( + data: LatLng[], +): Promise< + false | { way_id: number; distance_way: number; length_way: number }[] +> { + const request = { + shape: data.map((v) => ({ lat: v.lat, lon: v.lng })), + costing: 'auto', + shape_match: 'map_snap', + filters: { + attributes: [ + 'edge.way_id', + 'edge.length', + 'matched.distance_from_trace_point', + 'matched.point', + 'matched.type', + 'matched.edge_index', + 'matched.distance_along_edge', + ], + action: 'include', + }, + }; + + return fetch('http://localhost:8002/trace_attributes', { + method: 'POST', + body: JSON.stringify(request), + headers: { 'Content-Type': 'application/json' }, + }).then(async (res) => { + const json = await res.json(); + + if (json.error) return false; + + return json.matched_points.map( + (p: { + edge_index: number; + distance_along_edge: number; + lat: number; + lon: number; + }) => ({ + way_id: json.edges[p.edge_index]?.way_id, + distance_way: + p.distance_along_edge * json.edges[p.edge_index]?.length * 1000, + coordinates: { lat: p.lat, lng: p.lon }, + }), + ); + }); +}