Add valhalla in dynatest importation

backend/package.json

@@ -37,6 +37,7 @@
     "nestjs-knex": "^2.0.0",
     "papaparse": "^5.4.1",
     "pg": "^8.11.3",
+    "polyline": "^0.2.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
     "sharp": "^0.33.0",

backend/src/roads/road.controller.ts

@@ -1,11 +1,4 @@
-import {
-  BadRequestException,
-  Body,
-  Controller,
-  Get,
-  Param,
-  Post,
-} from '@nestjs/common';
+import { Body, Controller, Get, Post } from '@nestjs/common';
 import { RoadService } from './road.service';
 import { OSMWayId } from '../models';
 
@@ -57,17 +50,4 @@ export class RoadController {
 
     return { data: data, geometry: geometry };
   }
-
-  /**
-   * Add a specific road to the database using the OSM id of a way in the road.
-   * **It might get removed in the future.**
-   *
-   * @param id the OSM id of a way in the road
-   * @author Kerbourc'h
-   */
-  @Get('getorcreateway/:id')
-  async insertRoad(@Param('id') id: string) {
-    if (isNaN(Number(id))) throw new BadRequestException('Invalid id');
-    return await this.service.getOrCreateWay(id);
-  }
 }

backend/src/roads/road.service.ts

@@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
 import { InjectConnection, Knex } from 'nestjs-knex';
 import { IWay, Measurement, Way } from '../tables';
 import { LatLng, MeasurementType, OSMWayId, Road } from '../models';
-import { getOSMWaysInARoad } from './osm';
 import { constructNavigationTable, findLongestBranch } from './roads';
 
 @Injectable()
@@ -11,124 +10,6 @@ export class RoadService {
     @InjectConnection('group-d') private readonly knex_groupd: Knex,
   ) {}
 
-  /**
-   * Insert a way in the database.
-   * @param way the way to insert
-   * @returns the inserted way (in an array)
-   *
-   * @author Kerbourc'h
-   */
-  async insertWay(way: IWay) {
-    const { section_geom, ...rest } = way;
-
-    return Way(this.knex_groupd)
-      .insert({
-        ...rest,
-        section_geom: this.knex_groupd.raw('ST_GeomFromGeoJSON(?)', [
-          JSON.stringify(section_geom),
-        ]),
-      })
-      .onConflict('osm_id')
-      .merge() // Force to update because returning doesn't work with ignore()
-      .returning([
-        'id',
-        'way_name',
-        'osm_id',
-        'node_start',
-        'node_end',
-        'length',
-        this.knex_groupd.raw(
-          'ST_AsGeoJSON(section_geom)::json as section_geom',
-        ),
-        'isoneway',
-      ]);
-  }
-
-  /**
-   * Get or create the ways corresponding to the OSMWayIds.
-   * You should definitely use this function instead of getOrCreateWay if you
-   * have multiple ways to get.
-   * This executes the queries sequentially to avoid useless queries to OSM
-   * and the database. This is especially useful when the ways are in the same road.
-   *
-   * @param wayIds
-   *
-   * @author Kerbourc'h
-   */
-  async getOrCreateWays(wayIds: OSMWayId[]) {
-    let tasks: any = wayIds.map((id, index) =>
-      index === 0 ? this.getOrCreateWay(id).then((value: IWay) => [value]) : id,
-    );
-
-    return await tasks.reduce(async (cur: Promise<IWay[]>, next: string) => {
-      return cur.then(async (value: IWay[]) => {
-        if (((value.length / tasks.length) * 100) % 10 === 0)
-          console.info(
-            'Importation status',
-            `${(value.length / tasks.length) * 100}%`,
-          );
-
-        return [...value, await this.getOrCreateWay(next)];
-      });
-    });
-  }
-
-  /**
-   * Will get the Way corresponding to the OSMWayId either from the database if
-   * available or from OSM and insert the whole road containing it in the database.
-   *
-   * @param OSMWayId the osm id of a way in the road
-   *
-   * @author Kerbourc'h
-   */
-  async getOrCreateWay(OSMWayId: OSMWayId): Promise<IWay> {
-    const way = await Way(this.knex_groupd)
-      .select(
-        'id',
-        'way_name',
-        'osm_id',
-        'node_start',
-        'node_end',
-        'length',
-        this.knex_groupd.raw(
-          'ST_AsGeoJSON(section_geom)::json as section_geom',
-        ),
-        'isoneway',
-      )
-      .where('osm_id', OSMWayId)
-      .limit(1);
-
-    // If the way doesn't exist in the database, fetch the road from OSM
-    // and insert it in the database. Then return the way.
-    if (way.length === 0) {
-      console.info(
-        `Way ${OSMWayId} doesn't exist in the database, fetching road from OSM.`,
-      );
-      return await getOSMWaysInARoad(OSMWayId, async (data): Promise<IWay> => {
-        if (data.length > 0) {
-          console.info(
-            `Inserting road containing ${data.length} ways in the database.`,
-          );
-          return await Promise.all(data.map((way) => this.insertWay(way))).then(
-            async (result): Promise<IWay> => {
-              const ways = result.flat();
-              console.debug(
-                "Road's OSM Ids:",
-                ways.map((way) => way.osm_id).join(', '),
-              );
-              return ways.filter((way) => way.osm_id === OSMWayId)[0];
-            },
-          );
-        } else {
-          console.error('No way found in the road. Check the OSM Id.');
-          return null;
-        }
-      });
-    }
-
-    return way[0];
-  }
-
   /**
    * Query the geometry, node_start and node_end of each way and structure the data properly
    *

backend/src/upload/file.processor.ts

@@ -1,11 +1,7 @@
 import { InjectQueue, Process, Processor } from '@nestjs/bull';
 import { Job, Queue } from 'bull';
 import * as process from 'process';
-import {
-  extract_measurements_data,
-  extractCoordinatesFromRSP,
-  find_surveys,
-} from './upload';
+import { extract_measurements_data, find_surveys } from './upload';
 import { UploadService } from './upload.service';
 import { extract_dashcam_image_data, extract_road_image_data } from './image';
 import * as path from 'path';
@@ -92,39 +88,51 @@ export class FileProcessor {
    */
   @Process('process-file')
   async handleFileProcessing(job: Job<{ filePath: string }>) {
+    const printJobInfo = (...args: any[]) => {
+      console.log(`[${job.id} ${job.toJSON()['progress']}%]`, ...args);
+    };
+
+    const printJobError = (...args: any[]) => {
+      console.error(`[${job.id} ${job.toJSON()['progress']}%]`, ...args);
+    };
+
     try {
       const { filePath } = job.data;
       const debug = process.env.IMPORT_DEBUG === 'true';
-      console.log(`Processing file: ${filePath} (job id: ${job.id})`);
+      printJobInfo(`Processing file: ${filePath}`);
 
       //here we make sure that there is at least one RSP file and a HDC directory
       let surveys = find_surveys(filePath, debug);
       if (debug) {
-        console.debug(surveys);
+        printJobInfo(surveys);
       }
 
       if (surveys.length == 0) {
-        if (debug) {
-          console.log('No valid data found in directory: ' + filePath);
-        }
+        printJobInfo('No valid data found in directory: ' + filePath);
       }
 
       // TODO: split process here instead of for the all zip file (one process per survey)
       for (let i = 0; i < surveys.length; i++) {
-        // find the geometry of the survey
-        surveys[i].geometry = extractCoordinatesFromRSP(surveys[i].RSP);
-
         // upload the survey data and get the id back
         surveys[i].fk_survey_id = await this.service.db_insert_survey_data(
           surveys[i],
           debug,
         );
 
         const data = extract_measurements_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], data))) {
+          printJobError('Failed to map match data.');
+        }
+
         const roadImages = extract_road_image_data(surveys[i], debug);
-        const dashcameraImages = extract_dashcam_image_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], roadImages))) {
+          printJobError('Failed to map match road images.');
+        }
 
-        // TODO(Seb-sti1): (when rest working) add valhalla here
+        const dashcameraImages = extract_dashcam_image_data(surveys[i], debug);
+        if (!(await this.service.mapMatch(surveys[i], dashcameraImages))) {
+          printJobError('Failed to map match dashcam images.');
+        }
 
         await job.progress(65);
 
@@ -159,20 +167,20 @@ export class FileProcessor {
       try {
         const tempFolderPath = path.dirname(job.data.filePath);
         fs.rmSync(tempFolderPath, { recursive: true });
-        console.log(`Deleted extracted folder: ${job.data.filePath}`);
+        printJobInfo(`Deleted extracted folder: ${job.data.filePath}`);
       } catch (error) {
-        console.error(
+        printJobError(
           `Error deleting extracted folder: ${job.data.filePath}`,
           error,
         );
       }
-      console.log(
+      printJobInfo(
         `File processed successfully and imported into Database: ${filePath}`,
       );
       await job.progress(100);
       console.log(`Job ${job.id} completed`);
     } catch (error) {
-      console.error(`Error processing file: ${job.data.filePath}`, error);
+      printJobError(`Error processing file: ${job.data.filePath}`, error);
     }
   }
 }