Add Valhalla

backend/package.json

@@ -37,6 +37,7 @@
     "nestjs-knex": "^2.0.0",
     "papaparse": "^5.4.1",
     "pg": "^8.11.3",
+    "polyline": "^0.2.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
     "sharp": "^0.33.0",

backend/src/upload/file.processor.ts

@@ -88,21 +88,27 @@ export class FileProcessor {
    */
   @Process('process-file')
   async handleFileProcessing(job: Job<{ filePath: string }>) {
+    const printJobInfo = (...args: any[]) => {
+      console.log(`[${job.id} ${job.toJSON()['progress']}%]`, ...args);
+    };
+
+    const printJobError = (...args: any[]) => {
+      console.error(`[${job.id} ${job.toJSON()['progress']}%]`, ...args);
+    };
+
     try {
       const { filePath } = job.data;
       const debug = process.env.IMPORT_DEBUG === 'true';
-      console.log(`Processing file: ${filePath} (job id: ${job.id})`);
+      printJobInfo(`Processing file: ${filePath}`);
 
       //here we make sure that there is at least one RSP file and a HDC directory
       let surveys = find_surveys(filePath, debug);
       if (debug) {
-        console.debug(surveys);
+        printJobInfo(surveys);
       }
 
       if (surveys.length == 0) {
-        if (debug) {
-          console.log('No valid data found in directory: ' + filePath);
-        }
+        printJobInfo('No valid data found in directory: ' + filePath);
       }
 
       // TODO: split process here instead of for the all zip file (one process per survey)
@@ -114,11 +120,19 @@ export class FileProcessor {
         );
 
         const data = extract_measurements_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], data))) {
+          printJobError('Failed to map match data.');
+        }
 
         const roadImages = extract_road_image_data(surveys[i], debug);
-        const dashcameraImages = extract_dashcam_image_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], roadImages))) {
+          printJobError('Failed to map match road images.');
+        }
 
-        // TODO(Seb-sti1): (when rest working) add valhalla here
+        const dashcameraImages = extract_dashcam_image_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], dashcameraImages))) {
+          printJobError('Failed to map match dashcam images.');
+        }
 
         await job.progress(65);
 
@@ -153,20 +167,20 @@ export class FileProcessor {
       try {
         const tempFolderPath = path.dirname(job.data.filePath);
         fs.rmSync(tempFolderPath, { recursive: true });
-        console.log(`Deleted extracted folder: ${job.data.filePath}`);
+        printJobInfo(`Deleted extracted folder: ${job.data.filePath}`);
       } catch (error) {
-        console.error(
+        printJobError(
           `Error deleting extracted folder: ${job.data.filePath}`,
           error,
         );
       }
-      console.log(
+      printJobInfo(
         `File processed successfully and imported into Database: ${filePath}`,
       );
       await job.progress(100);
       console.log(`Job ${job.id} completed`);
     } catch (error) {
-      console.error(`Error processing file: ${job.data.filePath}`, error);
+      printJobError(`Error processing file: ${job.data.filePath}`, error);
     }
   }
 }

backend/src/upload/upload.service.ts

@@ -1,6 +1,11 @@
 import { Injectable } from '@nestjs/common';
 import { InjectConnection, Knex } from 'nestjs-knex';
-import { SurveyImage, SurveyRoadParameters, SurveyStructure } from './upload';
+import {
+  SurveyData,
+  SurveyImage,
+  SurveyRoadParameters,
+  SurveyStructure,
+} from './upload';
 import { join } from 'path';
 import { copyFileSync, existsSync, mkdirSync } from 'fs';
 
@@ -9,6 +14,12 @@ import * as sharp from 'sharp';
 import { OSMWayId } from '../models';
 import { IWay, Way } from '../tables';
 import { getOSMWaysInARoad } from './osm';
+import {
+  distanceFromWayBeginningToShapePoint,
+  Edge,
+  getGpsPointAtDistance,
+  valhalla,
+} from './valhalla';
 
 @Injectable()
 export class UploadService {
@@ -199,30 +210,27 @@ export class UploadService {
 
   /**
    * Get or create the ways corresponding to the OSMWayIds.
-   * You should definitely use this function instead of getOrCreateWay if you
-   * have multiple ways to get.
+   *
+   * You should definitely use this function instead of getOrCreateWay if you have multiple ways that
+   * are likely on the same road.
+   *
    * This executes the queries sequentially to avoid useless queries to OSM
    * and the database. This is especially useful when the ways are in the same road.
    *
    * @param wayIds
    *
    * @author Kerbourc'h
    */
-  async getOrCreateWays(wayIds: OSMWayId[]) {
+  async getOrCreateWays(wayIds: OSMWayId[]): Promise<IWay[]> {
     let tasks: any = wayIds.map((id, index) =>
       index === 0 ? this.getOrCreateWay(id).then((value: IWay) => [value]) : id,
     );
 
     return await tasks.reduce(async (cur: Promise<IWay[]>, next: string) => {
-      return cur.then(async (value: IWay[]) => {
-        if (((value.length / tasks.length) * 100) % 10 === 0)
-          console.info(
-            'Importation status',
-            `${(value.length / tasks.length) * 100}%`,
-          );
-
-        return [...value, await this.getOrCreateWay(next)];
-      });
+      return cur.then(async (value: IWay[]) => [
+        ...value,
+        await this.getOrCreateWay(next),
+      ]);
     });
   }
 
@@ -234,7 +242,7 @@ export class UploadService {
    *
    * @author Kerbourc'h
    */
-  async getOrCreateWay(OSMWayId: OSMWayId): Promise<IWay> {
+  async getOrCreateWay(OSMWayId: OSMWayId): Promise<IWay> | null {
     const way = await Way(this.knex_group_d)
       .select(
         'id',
@@ -281,4 +289,98 @@ export class UploadService {
 
     return way[0];
   }
+
+  /**
+   * Do the map matching of the data using valhalla.
+   * The result will directly be stored in the data array.
+   *
+   * @author Kerbourc'h
+   */
+  async mapMatch(survey: SurveyStructure, data: SurveyData[]) {
+    let geometry = survey.geometry;
+
+    // To have a good result for valhalla, first the point at the
+    // same distance from the beginning of the survey are removed
+    // the following arrays will allow to find back the corresponding data
+    let dataToPositionIndex: number[] = [];
+    let distances: number[] = [];
+
+    for (let i = 0; i < data.length; i++) {
+      const exist = distances.indexOf(data[i].distance_survey);
+
+      if (exist === -1) {
+        distances.push(data[i].distance_survey);
+        dataToPositionIndex.push(distances.length - 1);
+      } else {
+        dataToPositionIndex.push(exist);
+      }
+    }
+
+    // find the gps coordinates of the distances along the survey
+    let positions = distances.map((d) =>
+      getGpsPointAtDistance(geometry, d / 1000),
+    );
+
+    // use valhalla to map match that data (by construction matched_points.length === positions.length)
+    const matched = await valhalla(positions);
+    if (!matched) {
+      return false;
+    }
+
+    // get or create the ways (also inserting the road if needed) using osm
+    const ways = await this.getOrCreateWays(
+      matched.edges.map((m) => String(m.way_id)),
+    );
+
+    // for each edge, pair it with the corresponding way
+    // and determine the direction of the edge along the way and
+    // the distance from the beginning of the way to the beginning of the edge
+    const edges: Edge[] = matched.edges.map((e, idx) => {
+      // calculate the distance from the beginning of the way to the beginning of the edge
+      const distance_way_start_edge_start =
+        distanceFromWayBeginningToShapePoint(
+          e.begin_shape,
+          ways[idx].section_geom.coordinates.map((c) => ({
+            lat: c[1],
+            lng: c[0],
+          })),
+        );
+
+      // calculate the distance from the beginning of the way to the end of the edge
+      const distance_way_start_edge_end = distanceFromWayBeginningToShapePoint(
+        e.end_shape,
+        ways[idx].section_geom.coordinates.map((c) => ({
+          lat: c[1],
+          lng: c[0],
+        })),
+      );
+
+      return {
+        beginning_from_way_start: distance_way_start_edge_start,
+        way: ways[idx],
+        // apply a minus if the edge is not in the same direction as the way
+        length:
+          distance_way_start_edge_start < distance_way_start_edge_end
+            ? e.length
+            : -e.length,
+      };
+    });
+
+    for (let i = 0; i < data.length; i++) {
+      // find the corresponding matched data using dataToPositionIndex
+      const matched_data = matched.matched_points[dataToPositionIndex[i]];
+
+      // find corresponding edge
+      const edge = edges[matched_data.edge_index];
+
+      // edit data
+      data[i].fk_way_id = edge.way.id;
+      data[i].distance_way =
+        edge.beginning_from_way_start +
+        edge.length * matched_data.distance_ratio_along_edge;
+      data[i].position = matched_data.coordinates;
+    }
+
+    return true;
+  }
 }

backend/src/upload/upload.ts

@@ -194,10 +194,15 @@ export function extractCoordinatesFromRSP(rspFilePath: string): LatLng[] {
 
   for (let i = 0; i < datas.length; i++) {
     const row = datas[i];
-    coordinates.push({
-      lng: Number(`${parseFloat(row[6])}`),
-      lat: Number(`${parseFloat(row[5])}`),
-    });
+    // to fix bugged RSP file of Ballerup
+    const exist = coordinates.find(
+      (c) => c.lat == parseFloat(row[5]) && c.lng == parseFloat(row[6]),
+    );
+    if (!exist)
+      coordinates.push({
+        lng: Number(`${parseFloat(row[6])}`),
+        lat: Number(`${parseFloat(row[5])}`),
+      });
   }
 
   return coordinates;