diff --git a/backend/src/lira_importing/lira_importing_functions.js b/backend/src/lira_importing/lira_importing_functions.js new file mode 100644 index 00000000..1a522571 --- /dev/null +++ b/backend/src/lira_importing/lira_importing_functions.js @@ -0,0 +1,566 @@ +const { Console, debug } = require('console'); +const fs = require('fs'); +const readline = require('readline'); +const { pool } = require('./db.js'); +const Papa = require('papaparse'); + +const { + db_insert_survey_data, + liradb_get_all_trips, + liradb_get_all_coverage_by_fk_trip_id, + liradb_get_multilinestring_from_coverage_by_uuid, + liradb_get_timestamp, + db_check_if_surveyID_exists, + liradb_get_rowCount_and_all_uuids_for_coverage, + liradb_get_corverage_entry_by_uuid, + liradb_get_way_entry_by_uuid, + db_check_osm_id_and_get_uuid_in_way_tabel, + liradb_get_end_time_by_fk_trip_id, + liradb_get_coverage_values_by_fk_coverage_id, + liradb_get_type_and_value_for_coverage_values_entry_by_uuid, + db_insert_lira_data_in_measurement_table, +} = require('./lira_query_functions.js'); + +const { exit } = require('process'); +const internal = require('stream'); +const path = require('path'); + +const testFolder1 = './MFV Motorvej O4 2022'; +const testFolder2 = './MFV Ballerup 2023'; +const testFolder3 = './MFV Saudi Arabia 5051-299'; +const testFolder4 = './the_big_null_folder'; + +const HDCtestFolder = './MFV Ballerup 2023/Test2/HDC'; +const xmlTestFile = + './MFV Ballerup 2023/Test2/HDC/LcmsSurvey_3050354664_000089.xml'; + +/* These arrays and varibel are only used for the trips functions */ +let tripArr = []; +let trip_timestampArr = []; +let conv_fk_tripArr = []; +let trips_section_geomArr = []; +const lira_start_value = 1404000000; +let created_survey_idArr = []; +let fk_survey_idArr = []; + +/* These global arrays and varibels are used for the_new_controle_function*/ +let rowCount = -1; +let all_coverage_uuidArr = []; +let buff_distance01 = -1; +let buff_distance02 = -1; +let buff_fk_trip_id = -1; +let buff_fk_way_id = -1; +let buff_osm_id = -1; +let buff_length = -1; +let buff_osm_id_uuid = -1; +let buff_distance_way = -1; +let buff_lat_mapped = -1; +let buff_lon_mapped = -1; +let buff_timestamp = -1; +let buff_coverage_values_uuidArr = []; +let buff_rowCount = -1; +let buff_type = []; +let buff_value = []; + +the_new_controle_function(true); + +/* Function under here */ + +/** + * This function uses the liradb_get_all_trips function to get all entries uuid from the lira database, and put in into the global tripArr. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_all_trips(debug) { + const buff = liradb_get_all_trips(debug); + tripArr = await buff; +} + +/** + * This function uses the liradb_get_all_coverage_by_fk_trip_id function to get all entires that has a specific fk_trip_id, and sort in into the global conv_fk_tripArr[*1][*2]. It's a recursive function that runs for the length of the global tripArr. *1 is the uuid for trip entires. *2 is the uuid foor the coverage entires with the fk_trip_id corresponding to trip uuid. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_all_trips_coverage(debug) { + //We need var i, might look redudant but if its not there, the function won't be async and won't work. + var i; + for (i = 0; i < tripArr.length; i++) { + const buff = await liradb_get_all_coverage_by_fk_trip_id(tripArr[i], debug); + conv_fk_tripArr[i] = buff; + } +} + +/** + * This function gets all section_geom for a trip and create one long section_geom/multilinestring. It puts the createt section_geom in the trips_section_geom array, at the index corrosponding to tripArr index. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function liradb_get_section_geom_from_coverage(debug) { + var i; + var j; + var stringBuff; + var stringBuffArr = []; + for (i = 0; i < conv_fk_tripArr.length; i++) { + stringBuff = []; + stringBuffArr = []; + for (j = 0; j < conv_fk_tripArr[i].length; j++) { + const buff = await liradb_get_multilinestring_from_coverage_by_uuid( + conv_fk_tripArr[i][j], + false, + ); + stringBuff = await buff.split(' '); + stringBuffArr.push([ + Number(`${parseFloat(await stringBuff[0])}`), + Number(`${parseFloat(await stringBuff[1])}`), + ]); + } + trips_section_geomArr[i] = stringBuffArr; + if (debug) { + console.log('Data in the trips_section_geom at index ' + i + ': '); + console.log(trips_section_geomArr[i]); + console.log('-----------------------------------'); + } + } +} + +/** + * This function uses the liradb_get_timestamp function to get the timestamp for all trips, and sort in into the global trip_timestampArr. It's a recursive functino that runs for the length of the global tripArr. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_all_timestamps(debug) { + //We need var i, might look redudant but if its not there, the function won't be async and won't work. + var i; + for (i = 0; i < tripArr.length; i++) { + const buff = await liradb_get_timestamp(tripArr[i], debug); + trip_timestampArr[i] = buff; + } +} + +/** + * This function creates random survey id's and puts it them in the created_survey_idArr array at the index corrosponding to the tripsArr array. This function uses the db_check_if_surveyID_exists to check if a random genareted survey_id allready exists in the survey table. It's a recursive functino that runs for the length of the global tripArr. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function create_and_check_survey_id(debug) { + var i; + for (i = 0; i < tripArr.length; i++) { + kevin = true; + while (kevin) { + let buff = lira_start_value + Math.floor(Math.random() * 1000000); + if (!(await db_check_if_surveyID_exists(buff, debug))) { + created_survey_idArr[i] = buff; + kevin = false; + if (debug) { + console.log( + 'Value in created_survey_idArr index ' + + i + + ': ' + + created_survey_idArr[0], + ); + console.log('-----------------------------------'); + } + } + } + } +} + +/** + * This function uses the formatet trip data from the lira database, and uploads it all to the survey tabel in our database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function upload_lira_data_to_survey_table(debug) { + //We need var i, might look redudant but if its not there, the function won't be async and won't work. + var i; + for (i = 0; i < tripArr.length; i++) { + const buff = db_insert_survey_data( + JSON.stringify({ + type: 'MultiLineString', + coordinates: [trips_section_geomArr[i]], + }), + trip_timestampArr[i], + created_survey_idArr[i], + 'Lira', + false, + ); + + fk_survey_idArr[i] = await buff; + + if (debug) { + console.log('ID from the database: ' + fk_survey_idArr[i]); + console.log('Local variables for dataset: ' + DirArr[i]); + console.log('SurveyID for dataset: ' + local_surveyID); + console.log('date for dataset: ' + local_date); + console.log('section_geom: ' + section_geom); + console.log('-----------------------------------'); + } + } +} + +/** + * This function uses query function liradb_get_rowCount_and_uuids_for_coverage to get the data and then sort it. It modifies the global number rowCount and array all_coverage_uuidArr. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_and_sort_lira_rowCount_and_all_uuids_for_coverage(debug) { + //Query function that returns the result of the query + const res_buff = await liradb_get_rowCount_and_all_uuids_for_coverage(debug); + + rowCount = res_buff.rowCount; + + for (i = 0; i < rowCount; i++) { + all_coverage_uuidArr[i] = res_buff.rows[i].id; + } + + if (debug) { + console.log('The rowCount is: ' + rowCount); + console.log('uuid in all_coverage_uuidArr[0]: ' + all_coverage_uuidArr[0]); + console.log( + 'uuid in all_coverage_uuidArr[rowCount - 1]: ' + + all_coverage_uuidArr[rowCount - 1], + ); + console.log('-----------------------------------'); + } +} + +/** + * This function uses query function liradb_get_one_corverage_entry_by_uuid to get the data then sort it. It modifies the global numbers buff_distance01, buff_distance02, buff_fk_trip_id and buff_fk_way_id. + * @param {number} uuid is the uuid we want the query function to get. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_and_sort_lira_corverage_entry(uuid, debug) { + const res_buff = await liradb_get_corverage_entry_by_uuid(uuid, debug); + + buff_distance01 = res_buff.rows[0].distance01; + buff_distance02 = res_buff.rows[0].distance02; + buff_fk_trip_id = res_buff.rows[0].fk_trip_id; + buff_fk_way_id = res_buff.rows[0].fk_way_id; + buff_lat_mapped = res_buff.rows[0].lat_mapped; + buff_lon_mapped = res_buff.rows[0].lon_mapped; + + if (debug) { + console.log('Data after sort for coverage entry uuid: ' + uuid); + console.log('distance01: ' + buff_distance01); + console.log('distance02: ' + buff_distance02); + console.log('fk_trip_id: ' + buff_fk_trip_id); + console.log('fk_way_id: ' + buff_fk_way_id); + console.log('-----------------------------------'); + } +} + +/** + * This function uses query function liradb_get_one_way_entry_by_uuid to get the data then sort it. It modifies the global numbers buff_osm_id and buff_lenght. + * @param {number} uuid is the uuid we want the query function to get. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_and_sort_lira_way_entry(uuid, debug) { + const res_buff = await liradb_get_way_entry_by_uuid(uuid, debug); + buff_osm_id = res_buff.rows[0].OSM_Id; + buff_length = res_buff.rows[0].lenght; + + if (debug) { + console.log('Data after sort for way entry uuid: ' + uuid); + console.log('buff_osm_id: ' + buff_osm_id); + console.log('buff_lenght: ' + buff_length); + console.log('-----------------------------------'); + } +} + +/** + * This function uses query function db_check_osm_id_and_get_uuid_in_way_tabel to search and get data from our database. It then sorts the data and return false if the data is valid. This function is meant to be used as the skip if we don't have the osm_id in our database. So true, we skip this coverage entry and false we use this coverage entry. + * @param {number} osm_id is the osm_id we want the query function to seach for. + * @param {number} length is the length for the osm_id we got from the liradatabase. This is used for an extra check that our osm data is equal'ish to the lira osm data. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @returns a boolean thats false if the osm_id exists in our database, else true. + * @author Vejlgaard + */ +async function get_check_and_sort_db_way_entry(osm_id, length, debug) { + buff_osm_id_uuid = await db_check_osm_id_and_get_uuid_in_way_tabel( + osm_id, + length, + debug, + ); + + if (buff_osm_id_uuid != -1) { + return false; + } else { + return true; + } +} + +/** + * This is a very simpel function where the simpel calulations for distance_way is done. Uses the formula: ((distance01 + distance02) / 2) * buff_length; + * @param {number} distance01 is distance 1 from the coverage entry. + * @param {number} distance02 is distance 2 from the coverage entry. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @returns the result of the calculation as a number. + * @author Vejlgaard + */ +function simpel_distance_way(distance01, distance02, debug) { + buff = ((distance01 + distance02) / 2) * buff_length; + if (debug) { + console.log('Simpel distance_way: ' + buff); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function uses the query function liradb_get_coverage_values_by_fk_coverage_id to get all coverage_valures with the fk_coverage_key equal to the given uuid, from the lira database. It then sorts the data into the global number buff_rowCount and global array buff_coverage_values_uuidArr. + * @param {number} uuid is the uuid we want the query function to get. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_and_sort_lira_coverage_values_by_fk_coverage_id( + uuid, + debug, +) { + //Query function that returns the result of the query + const res_buff = await liradb_get_coverage_values_by_fk_coverage_id( + uuid, + debug, + ); + + buff_rowCount = res_buff.rowCount; + + for (i = 0; i < buff_rowCount; i++) { + buff_coverage_values_uuidArr[i] = res_buff.rows[i].id; + } + + if (debug) { + console.log('The rowCount is: ' + buff_rowCount); + console.log( + 'uuid in buff_coverage_values_uuidArr[0]: ' + + buff_coverage_values_uuidArr[0], + ); + console.log( + 'uuid in buff_coverage_values_uuidArr[rowCount - 1]: ' + + buff_coverage_values_uuidArr[buff_rowCount - 1], + ); + console.log('-----------------------------------'); + } +} + +/** + * This function uses the query function liradb_get_type_and_value_for_coverage_values_entry_by_uuid to get type and value for all the entries in buff_coverage_values_uuidArr. It then sorts the data into global array buff_type and buff_value. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function get_and_sort_lira_corverage_value_type_and_value(debug) { + var j; + for (j = 0; j < buff_rowCount; j++) { + const res_buff = + await liradb_get_type_and_value_for_coverage_values_entry_by_uuid( + buff_coverage_values_uuidArr[j], + debug, + ); + buff_type[j] = res_buff.rows[0].type; + //We have to round to only 15 decimals, our database can max have 15 decimals. We might loss 2 or 3 decimals at the end. + buff_value[j] = res_buff.rows[0].value.toFixed(15); + if (debug) { + console.log('The buff_type is: ' + buff_type[j] + ' for index: ' + i); + console.log('The buff_value is: ' + buff_value[j] + ' for index: ' + i); + console.log('-----------------------------------'); + } + } +} + +/** + * This function takes a string and returns the type as number. This function is a big switch statment, basicly a lookup tabel for the content of AvailArr. Only implementet strings for liravis type data. + * @param {string} type the type as a string from the liravis database. + * @returns the type index as a number. + * @author Vejlgaard + */ +function give_index_number_for_lira_coverage_value(type) { + switch (type) { + case 'E_norm': + return 19; + case 'KPI': + return 20; + case 'Mu': + return 21; + case 'DI': + return 22; + case 'E_whl': + return 23; + case 'E_areo': + return 24; + case 'E_whl_std': + return 25; + case 'E_tire_std': + return 26; + case 'E_tire': + return 27; + case 'E_ineratia_slope': + return 28; + case 'E_inertia_slope_std': + return 29; + case 'E_areo_std': + return 30; + case 'E_norm_std': + return 31; + case 'mu_std': + return 32; + case 'IRI': + return 33; + case 'IRI_new': + return 34; + case 'E_brk': + return 35; + case 'E_brk_std': + return 36; + case 'E_inertia_slope_std': + return 37; + default: + return 38; + } +} + +/** + * This function uses the query function db_insert_lira_data_in_measurement_table to upload entries to the measument tabel in our database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function upload_all_coverage_values_for_coverage_to_db(debug) { + var j = 0; + + for (j; j < buff_rowCount; j++) { + await db_insert_lira_data_in_measurement_table( + give_index_number_for_lira_coverage_value(buff_type[j]), + buff_value[j], + buff_timestamp, + buff_distance_way, + buff_osm_id_uuid, + buff_lat_mapped, + buff_lon_mapped, + debug, + ); + if (debug) { + console.log( + 'Uploaded coverage_value: ' + j + ' to the measurment table in db', + ); + } + } +} + +/** + * This is the controle function for the whole process of importing, reformatning and uploading the lira data to our database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @author Vejlgaard + */ +async function the_new_controle_function(debug) { + //STEP 1 + await get_and_sort_lira_rowCount_and_all_uuids_for_coverage(debug); + + var i = 0; + + for (i; i < rowCount; i++) { + buff_distance01 = -1; + buff_distance02 = -1; + buff_fk_trip_id = -1; + buff_fk_way_id = -1; + buff_osm_id = -1; + buff_length = -1; + buff_osm_id_uuid = -1; + buff_distance_way = -1; + buff_lat_mapped = -1; + buff_lon_mapped = -1; + buff_timestamp = -1; + buff_coverage_values_uuidArr = []; + buff_rowCount = -1; + buff_type = []; + buff_value = []; + + //STEP 2 + await get_and_sort_lira_corverage_entry(all_coverage_uuidArr[i], debug); + //STEP 3 + await get_and_sort_lira_way_entry(buff_fk_way_id, debug); + //STEP 4 + if ( + await get_check_and_sort_db_way_entry(buff_osm_id, buff_length, debug) + ) { + continue; + } + //STEP 5 + buff_distance_way = simpel_distance_way( + buff_distance01, + buff_distance02, + debug, + ); + //STEP 6 + buff_timestamp = await liradb_get_end_time_by_fk_trip_id( + buff_fk_trip_id, + debug, + ); + + //STEP 7 + await get_and_sort_lira_coverage_values_by_fk_coverage_id( + all_coverage_uuidArr[i], + debug, + ); + //STEP 8 + await get_and_sort_lira_corverage_value_type_and_value(debug); + //STEP 9 + await upload_all_coverage_values_for_coverage_to_db(debug); + } +} + +/* +Content of AvailArr +0. Crack detection module +1. Rutting module +2. Macro-texture module +3. Lane marking module +4. Rumble strip detection module +5. Potholes module +6. Drop-off and curb module +7. Joint module for concrete pavement +8. Raveling module +9. Roughness module +10. Road geometry (slope and cross-slope) module +11. Water entrapment calculation +12. Shoving calculation +13. Pick-out detection module +14. Bleeding module +15. Sealed crack module +16. Manholes(man-made object-mmo) module +17. Patch detection module +18. Pumping detection module +19. E_norm +20. KPI +21. Mu +22. DI +23. E_whl +24. E_areo +25. E_whl_std +26. E_tire_std +27. E_tire +28. E_ineratia_slope +29. E_inertia_slope_std +30. E_areo_std +31. E_norm_std +32. mu_std +33. IRI +34. IRI_new +35. E_brk +36. E_brk_std +37. E_inertia_slope_std +*/ + +/* +The index of importent types: +0. Crack detection module +5. Potholes module +9. Roughness module +15. Sealed crack module +16. Manholes(man-made object-mmo) module +19. E_norm +20. KPI +21. Mu +22. DI +32. mu_std +33. IRI +*/ diff --git a/backend/src/lira_importing/lira_query_functions.js b/backend/src/lira_importing/lira_query_functions.js new file mode 100644 index 00000000..1858e9ba --- /dev/null +++ b/backend/src/lira_importing/lira_query_functions.js @@ -0,0 +1,477 @@ +const { Console } = require('console'); +const { pool } = require('./db'); +const { lirapool } = require('./liradb'); + +/** + * Gets all uuid from all entries in the tabel trips in the lira database through a query. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. + * @returns an array with all the entries uuid as strings, from the lira database table called "trips". + * @author Vejlgaard + */ +async function liradb_get_all_trips(debug) { + var buff = []; + try { + const res = await lirapool.query('SELECT * FROM trips'); + if (debug) { + console.log('There are: ' + res.rowCount + ' entries in the trips tabel'); + console.log('-----------------------------------'); + } + for (i = 0; i < res.rowCount; i++) { + buff[i] = res.rows[i].id; + } + } catch (error) { + console.error(error); + } + return buff; +} + +/** + * Gets all entries in the coverage table, in the lira database, that has the fk_trip_id given to the function. + * @param {number} fk_trip_id is the uuid of an entry in the trip tabel in the lira database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns an array with all the uuid as strings, of the entries that has the fk_trip_id. + * @author Vejlgaard + */ +async function liradb_get_all_coverage_by_fk_trip_id(fk_trip_id, debug) { + var buff = []; + try { + const res = await lirapool.query( + "SELECT * FROM coverage WHERE fk_trip_id='" + + fk_trip_id + + "' ORDER BY compute_time;", + ); + if (debug) { + console.log( + 'There are: ' + + res.rowCount + + ' entries in the condition_coverage tabel', + ); + console.log('-----------------------------------'); + } + for (i = 0; i < res.rowCount; i++) { + buff[i] = res.rows[i].id; + } + } catch (error) { + console.error(error); + } + return buff; +} + +/** + * Gets the section geometry as a string for a entry with the uuid, reformats it and returns it. + * @param {number} uuid is the uuid of the entry we want the section_geom from. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns a string of the section_geom in format: 12.559807556085179 55.71318741557432, 12.559526276286647 55.71355835839164. + * @author Vejlgaard + */ +async function liradb_get_multilinestring_from_coverage_by_uuid(uuid, debug) { + var buff; + try { + const res = await lirapool.query( + "SELECT ST_AsText(section_geom) FROM coverage WHERE id='" + uuid + "'", + ); + buff = res.rows[0].st_astext.split('(').pop().split(')')[0]; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('uuid search for: ' + uuid); + console.log('section_geom: ' + buff); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function gets the timestamp as a string for a entry with the uuid, reformats it and returns it. + * @param {number} uuid is the uuid of the entry we want the timestamp from. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns a string of the timestamp in format: YYYY/MM/DD HH24:MM:SS. + * @author Vejlgaard + */ +async function liradb_get_timestamp(uuid, debug) { + var buff; + try { + const res = await lirapool.query( + "SELECT TO_CHAR(end_time_utc, 'YYYY/MM/DD HH24:MM:SS') FROM trips WHERE id='" + + uuid + + "'", + ); + buff = res.rows[0].to_char; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('uuid search for in trip tabel: ' + uuid); + console.log('timestamp: ' + buff); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function checks if the given survey_id exists in the survey table and returns the answere as a boolean. + * @param {number} survey_id is the survey_id of the entry we want to check for. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns a boolean that is true if the survey_id exists and false if it dosent exists. + * @author Vejlgaard + */ +async function db_check_if_surveyID_exists(survey_id, debug) { + var buff; + try { + const res = await pool.query( + //"WHERE EXISTS (SELECT * FROM survey WHERE survey_id='" + survey_id + "');" + "SELECT exists (SELECT 1 FROM survey WHERE survey_id = '" + + survey_id + + "' LIMIT 1);", + ); + buff = res.rows[0].exists; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('survey_id search for: ' + survey_id); + console.log('exists: ' + buff); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function gets all entries where distance01 != 0 AND distance02 != 0 from the coverage tabel in lira database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the resulting query for sorting. + * @author Vejlgaard + */ +async function liradb_get_rowCount_and_all_uuids_for_coverage(debug) { + var buff = 0; + try { + const res = await lirapool.query( + 'SELECT * FROM coverage WHERE distance01 != 0 AND distance02 != 0', + ); + buff = res; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log( + 'rowCount for all valid entries in the coverage tabel: ' + buff.rowCount, + ); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function gets the entry, from the coverage table, with the uuid and returns the query. + * @param {number} uuid is the uuid of the entry we want the data from. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the resulting query for sorting. + * @author Vejlgaard + */ +async function liradb_get_corverage_entry_by_uuid(uuid, debug) { + var buff = 0; + try { + const res = await lirapool.query( + "SELECT * FROM coverage WHERE id='" + uuid + "'", + ); + buff = res; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('Data from query for coverage entry uuid: ' + uuid); + console.log('distance01: ' + buff.rows[0].distance01); + console.log('distance02: ' + buff.rows[0].distance02); + console.log('fk_trip_id: ' + buff.rows[0].fk_trip_id); + console.log('fk_way_id: ' + buff.rows[0].fk_way_id); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function gets the entry, from the ways table, with the uuid and returns the query. + * @param {number} uuid is the uuid of the entry we want the data from. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the resulting query for sorting. + * @author Vejlgaard + */ +async function liradb_get_way_entry_by_uuid(uuid, debug) { + var buff = 0; + try { + const res = await lirapool.query( + "SELECT * FROM ways WHERE id='" + uuid + "'", + ); + buff = res; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('Data from query for way entry uuid: ' + uuid); + console.log('osm_id: ' + buff.rows[0].OSM_Id); + console.log('length: ' + buff.rows[0].lenght); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function search for an entry with the given osm_id. Returns the uuid for the entry if found, else return -1. + * @param {number} osm_id is the osm_id of the entry we are seaching for. + * @param {number} buff_length is the length from the lira database. Is used to check that the length form lira database and our is the equal. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the uuid as a number if a entry is found with the osm_id, else return -1. + * @author Vejlgaard + */ +async function db_check_osm_id_and_get_uuid_in_way_tabel( + osm_id, + buff_length, + debug, +) { + var buff = 0; + try { + const res = await pool.query( + "SELECT * FROM way WHERE osm_id = '" + osm_id + "';", + ); + buff = -1; + if (res.rowCount != 0) { + if (res.rows[0].length.toFixed(2) == buff_length.toFixed(2)) { + buff = res.rows[0].id; + } + } + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('osm_id seached for in way tabel: ' + osm_id); + if (buff != -1) { + console.log('uuid found for osm_id: ' + buff); + console.log('-----------------------------------'); + } else { + console.log('No valid data in our database for osm_id'); + } + } + return buff; +} + +/** + * This function uses the fk_trip_id to finde a timestamp for the coverage entry. + * @param {number} fk_trip_id is the uuid for the trip we are seaching for. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the end_time for the trip that the coverage is from. + * @author Vejlgaard + */ +async function liradb_get_end_time_by_fk_trip_id(fk_trip_id, debug) { + var buff = 0; + try { + const res = await lirapool.query( + "SELECT TO_CHAR(end_time_utc, 'YYYY/MM/DD HH24:MM:SS') FROM trips WHERE id='" + + fk_trip_id + + "'", + ); + buff = res.rows[0].to_char; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('fk_trip_id seached for in trip tabel: ' + fk_trip_id); + console.log('timestamp: ' + buff); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function uses a uuid, from an coverage tabel entry, as fk_coverage_id to find all associated entries in the coverage_values tabel. + * @param {number} uuid is the fk_coverage_id of the entry/entries we want to get: + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the resulting query for sorting. + * @author Vejlgaard + */ +async function liradb_get_coverage_values_by_fk_coverage_id(uuid, debug) { + var buff = 0; + try { + const res = await lirapool.query( + "SELECT * FROM coverage_values WHERE fk_coverage_id = '" + uuid + "';", + ); + buff = res; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('fk_coverage_id seached for in coverage_values tabel: ' + uuid); + console.log( + 'rowCount for all valid entries in the coverage tabel: ' + buff.rowCount, + ); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * This function uses a uuid to get the type and value from an entry in the coverage_values tabel. + * @param {number} uuid is the uuid of the entry we want to get: + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @returns the resulting query for sorting. + * @author Vejlgaard + */ +async function liradb_get_type_and_value_for_coverage_values_entry_by_uuid( + uuid, + debug, +) { + var buff = 0; + try { + const res = await lirapool.query( + "SELECT * FROM coverage_values WHERE id = '" + uuid + "';", + ); + buff = res; + } catch (error) { + console.error(error); + } + + if (debug) { + console.log('id seached for in coverage_values tabel: ' + uuid); + console.log('-----------------------------------'); + } + return buff; +} + +/** + * Uploads one entry to the measurment table with the given parameters as data. This function is just a variation of the query function db_insert_measurement_data, meant for the liravis importing proces. + * @param {number} type_index the type of measurment, see readme or bottom of file for description of numbers. Can be null. + * @param {number} value the value for the measurment, can mean different things dependant on type of measurment. Can be null. + * @param {number} timestamp the date in format: year/month/day hour:minut:second.milisecond. Can be null. + * @param {number} distance_way the distance from the begining of the way. Can be null. + * @param {number} fk_way_id the automatically generated uuid from the database for the corresponding way entry in the survey table. Can't be null. + * @param {number} latitude the latitude mapped from the coverage table in the lira database. + * @param {number} longitude the longitude mapped from the coverage table in the lira database. + * @param {boolean} debug debug boolean for printing relevant information, and is false by default. It is highly recommended to set the debug boolean to false. + * @author Vejlgaard + */ +async function db_insert_lira_data_in_measurement_table( + type_index, + value, + timestamp, + distance_way, + fk_way_id, + latitude, + longitude, + debug, +) { + try { + const res = await pool.query( + 'INSERT INTO measurement (type_index, value, timestamp, distance_way, fk_way_id, latitude, longitude) VALUES ($1, $2, $3, $4, $5, $6, $7)', + [ + type_index, + value, + timestamp, + distance_way, + fk_way_id, + latitude, + longitude, + ], + ); + } catch (error) { + console.error(error); + } + if (debug) { + console.log('Data uploaded to the database: '); + console.log('type_index: ' + type_index); + console.log('value: ' + value); + console.log('timestamp: ' + timestamp); + console.log('distance_way: ' + distance_way); + console.log('fk_way_id: ' + fk_way_id); + console.log('latitude: ' + latitude); + console.log('longitude: ' + longitude); + console.log('-----------------------------------'); + } +} + +module.exports = { + liradb_get_all_trips: liradb_get_all_trips, + liradb_get_all_coverage_by_fk_trip_id: liradb_get_all_coverage_by_fk_trip_id, + liradb_get_multilinestring_from_coverage_by_uuid: + liradb_get_multilinestring_from_coverage_by_uuid, + liradb_get_timestamp: liradb_get_timestamp, + db_check_if_surveyID_exists: db_check_if_surveyID_exists, + liradb_get_rowCount_and_all_uuids_for_coverage: + liradb_get_rowCount_and_all_uuids_for_coverage, + liradb_get_corverage_entry_by_uuid: liradb_get_corverage_entry_by_uuid, + liradb_get_way_entry_by_uuid: liradb_get_way_entry_by_uuid, + db_check_osm_id_and_get_uuid_in_way_tabel: + db_check_osm_id_and_get_uuid_in_way_tabel, + liradb_get_end_time_by_fk_trip_id: liradb_get_end_time_by_fk_trip_id, + liradb_get_coverage_values_by_fk_coverage_id: + liradb_get_coverage_values_by_fk_coverage_id, + liradb_get_type_and_value_for_coverage_values_entry_by_uuid: + liradb_get_type_and_value_for_coverage_values_entry_by_uuid, + db_insert_lira_data_in_measurement_table: + db_insert_lira_data_in_measurement_table, +}; + +/* +Content of AvailArr +0. Crack detection module +1. Rutting module +2. Macro-texture module +3. Lane marking module +4. Rumble strip detection module +5. Potholes module +6. Drop-off and curb module +7. Joint module for concrete pavement +8. Raveling module +9. Roughness module +10. Road geometry (slope and cross-slope) module +11. Water entrapment calculation +12. Shoving calculation +13. Pick-out detection module +14. Bleeding module +15. Sealed crack module +16. Manholes(man-made object-mmo) module +17. Patch detection module +18. Pumping detection module +19. E_norm +20. KPI +21. Mu +22. DI +23. E_whl +24. E_areo +25. E_whl_std +26. E_tire_std +27. E_tire +28. E_ineratia_slope +29. E_inertia_slope_std +30. E_areo_std +31. E_norm_std +32. mu_std +33. IRI +34. IRI_new +35. E_brk +36. E_brk_std +37. E_inertia_slope_std +*/ + +/* +The index of importent types: +0. Crack detection module +5. Potholes module +9. Roughness module +15. Sealed crack module +16. Manholes(man-made object-mmo) module +19. E_norm +20. KPI +21. Mu +22. DI +32. mu_std +33. IRI +*/ diff --git a/backend/src/lira_importing/package-lock.json b/backend/src/lira_importing/package-lock.json new file mode 100644 index 00000000..6bb88765 --- /dev/null +++ b/backend/src/lira_importing/package-lock.json @@ -0,0 +1,204 @@ +{ + "name": "xml_test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "xml_test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "papaparse": "^5.4.1", + "pg": "^8.11.3", + "typescript": "^5.2.2", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/node": "^20.6.3" + } + }, + "node_modules/@types/node": { + "version": "20.6.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.6.3.tgz", + "integrity": "sha512-HksnYH4Ljr4VQgEy2lTStbCKv/P590tmPe5HqOnv9Gprffgv5WXAY+Y5Gqniu0GGqeTCUdBnzC3QSrzPkBkAMA==", + "dev": true + }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, + "node_modules/papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, + "node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/typescript": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/backend/src/lira_importing/package.json b/backend/src/lira_importing/package.json new file mode 100644 index 00000000..4452b6e7 --- /dev/null +++ b/backend/src/lira_importing/package.json @@ -0,0 +1,21 @@ +{ + "dependencies": { + "papaparse": "^5.4.1", + "pg": "^8.11.3", + "typescript": "^5.2.2", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/node": "^20.6.3" + }, + "name": "xml_test", + "description": "## Missing features", + "version": "1.0.0", + "main": "database_functions.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC" +} diff --git a/backend/src/lira_importing/readme.md b/backend/src/lira_importing/readme.md new file mode 100644 index 00000000..998a51b7 --- /dev/null +++ b/backend/src/lira_importing/readme.md @@ -0,0 +1,82 @@ +# readme + +**These functions are only meant to be used ones to import the lira database, reformat it and uploaded to our own database. For these scripts to work you need a copy off the db.js and liradb.js for the scripts to work. These files will not be uploaded to github.** + +## A list over function in lira_database_import_functions.js + +**This counts for all functions with debug boolean. The debug boolean is by default false, and the function can be called without a debug argument given.** + +- async function get_all_trips(debug) +- async function get_all_trips_coverage(debug) +- async function liradb_get_section_geom_from_coverage(debug) +- async function get_all_timestamps(debug) +- async function create_and_check_survey_id(debug) +- async function upload_lira_data_to_survey_table(debug) +- async function the_old_controle_function(debug) +- async function get_and_sort_lira_rowCount_and_all_uuids_for_coverage(debug) +- async function get_and_sort_lira_corverage_entry(uuid, debug) +- async function get_and_sort_lira_way_entry(uuid, debug) +- async function get_check_and_sort_db_way_entry(osm_id, length, debug) +- function simpel_distance_way(distance01, distance02, debug) +- async function get_and_sort_lira_coverage_values_by_fk_coverage_id(uuid, debug) +- async function get_and_sort_lira_corverage_value_type_and_value(debug) +- function give_index_number_for_lira_coverage_value(type) +- async function upload_all_coverage_values_for_coverage_to_db(debug) +- async function the_new_controle_function(debug) + +## A list over function in lira_query_functions.js + +- async function liradb_get_all_trips(debug) +- async function liradb_get_all_coverage_by_fk_trip_id(fk_trip_id, debug) +- async function liradb_get_multilinestring_from_coverage_by_uuid(uuid, debug) +- async function liradb_get_timestamp(uuid, debug) +- async function db_check_if_surveyID_exists(survey_id, debug) +- async function liradb_get_rowCount_and_all_uuids_for_coverage(debug) +- async function liradb_get_corverage_entry_by_uuid(uuid, debug) +- async function liradb_get_way_entry_by_uuid(uuid, debug) +- async function db_check_osm_id_and_get_uuid_in_way_tabel(osm_id, buff_length, debug) +- async function liradb_get_end_time_by_fk_trip_id(fk_trip_id, debug) +- async function liradb_get_coverage_values_by_fk_coverage_id(uuid, debug) +- async function liradb_get_type_and_value_for_coverage_values_entry_by_uuid(uuid, debug) +- async function db_insert_lira_data_in_measurement_table(type_index, value, timestamp, distance_way, fk_way_id, debug) + +## Content of AvailArr 0. Crack detection module + +0. Cracking Module +1. Rutting module +2. Macro-texture module +3. Lane marking module +4. Rumble strip detection module +5. Potholes module +6. Drop-off and curb module +7. Joint module for concrete pavement +8. Raveling module +9. Roughness module +10. Road geometry (slope and cross-slope) module +11. Water entrapment calculation +12. Shoving calculation +13. Pick-out detection module +14. Bleeding module +15. Sealed crack module +16. Manholes(man-made object-mmo) module +17. Patch detection module +18. Pumping detection module +19. E_norm +20. KPI +21. Mu +22. DI +23. E_whl +24. E_areo +25. E_whl_std +26. E_tire_std +27. E_tire +28. E_ineratia_slope +29. E_inertia_slope_std +30. E_areo_std +31. E_norm_std +32. mu_std +33. IRI +34. IRI_new +35. E_brk +36. E_brk_std +37. E_inertia_slope_std