Migrate data process from DynatestData repo

DTU-SE-Group-D · Dec 12, 2023 · c987fca · c987fca
1 parent 5141d8b
commit c987fca
Show file tree

Hide file tree

Showing 12 changed files with 2,369 additions and 50 deletions.
diff --git a/backend/package-lock.json b/backend/package-lock.json
diff --git a/backend/package.json b/backend/package.json
@@ -34,6 +34,7 @@
     "knex-postgis": "^0.14.3",
     "multer": "^1.4.5-lts.1",
     "nestjs-knex": "^2.0.0",
+    "papaparse": "^5.4.1",
     "pg": "^8.11.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1"
@@ -49,6 +50,7 @@
     "@types/jest": "^29.5.5",
     "@types/multer": "^1.4.10",
     "@types/node": "^20.6.2",
+    "@types/papaparse": "^5.3.14",
     "@types/supertest": "^2.0.12",
     "@typescript-eslint/eslint-plugin": "^6.7.0",
     "@typescript-eslint/parser": "^6.7.0",

diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts
@@ -20,6 +20,7 @@ import { ImageService } from './images/image.service';
 import { SurveyService } from './surveys/survey.service';
 import { SurveyController } from './surveys/survey.controller';
 import { UploadController } from './upload/upload.controller';
+import { UploadService } from './upload/upload.service';
 
 import { BullModule } from '@nestjs/bull';
 import { FileProcessor } from './upload/file.processor';
@@ -71,6 +72,7 @@ const database = (config: any, name: string) => {
     SurveyService,
     ImageService,
     FileProcessor,
+    UploadService,
   ],
 })
 export class AppModule {}
diff --git a/backend/src/tables.ts b/backend/src/tables.ts
@@ -36,7 +36,7 @@ export const Ways = (k: Knex) => k.from<IWays>('ways');
 //
 
 export interface IImage {
-  id?: number;
+  id?: string;
   fk_survey_id: string;
   distance_survey: number;
   image_path: string;
@@ -49,7 +49,7 @@ export interface IImage {
 export const Image = (k: Knex) => k.from<IImage>('image');
 
 export interface IMeasurement {
-  id?: number;
+  id?: string;
   fk_survey_id: string;
   distance_survey: number;
   type_index: MeasurementType;
@@ -62,17 +62,18 @@ export interface IMeasurement {
 export const Measurement = (k: Knex) => k.from<IMeasurement>('measurement');
 
 export interface ISurvey {
-  id?: number;
+  id?: string;
   section_geom: Geometry;
   timestamp: Date;
+  datasource: string;
   /** The Dynatest id of the surveys or null */
   survey_id: number;
 }
 
 export const Survey = (k: Knex) => k.from<ISurvey>('survey');
 
 export interface IWay<T = LineString> {
-  id?: number;
+  id?: string;
   way_name: string;
   osm_id: string; // This is because bigint is not supported by knex
   node_start: string; // See https://stackoverflow.com/questions/39168501/pg-promise-returns-integers-as-strings/39176670#39176670