From d460910c1943008d300460f47adf2370c35d33ae Mon Sep 17 00:00:00 2001 From: Seb-sti1 <65665540+seb-sti1@users.noreply.github.com> Date: Sun, 17 Dec 2023 20:49:15 +0100 Subject: [PATCH] Show upload status on frontend --- backend/src/upload/file.processor.ts | 34 +++- backend/src/upload/upload.controller.ts | 7 +- .../src/Components/Conditions/UploadPanel.tsx | 167 ++++++++++++++---- frontend/src/css/single_file_input.css | 2 +- frontend/src/css/upload_panel.css | 52 +++++- frontend/src/models/models.ts | 9 + frontend/src/queries/upload.ts | 7 +- 7 files changed, 223 insertions(+), 55 deletions(-) diff --git a/backend/src/upload/file.processor.ts b/backend/src/upload/file.processor.ts index 0a0d2afa..eddea541 100644 --- a/backend/src/upload/file.processor.ts +++ b/backend/src/upload/file.processor.ts @@ -46,8 +46,9 @@ export class FileProcessor { console.error(`Error unzipping file: ${filePath}`, error); }) .on('close', async () => { - await this.handleUnzippedContent(tempUnzipPath, filePath); await job.progress(60); + await this.handleUnzippedContent(tempUnzipPath, filePath); + await job.progress(100); }); } catch (error) { console.error(`Error processing file: ${job.data.filePath}`, error); @@ -103,6 +104,7 @@ export class FileProcessor { //here we make sure that there is at least one RSP file and a HDC directory let surveys = find_surveys(filePath, debug); + await job.progress(10); if (debug) { printJobInfo(surveys); } @@ -111,7 +113,6 @@ export class FileProcessor { printJobInfo('No valid data found in directory: ' + filePath); } - // TODO: split process here instead of for the all zip file (one process per survey) for (let i = 0; i < surveys.length; i++) { // upload the survey data and get the id back surveys[i].fk_survey_id = await this.service.db_insert_survey_data( @@ -120,21 +121,40 @@ export class FileProcessor { ); const data = extract_measurements_data(surveys[i], debug); + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (1 / 7) + 10, + ); + if (!(await this.service.mapMatch(surveys[i], data))) { printJobError('Failed to map match data.'); } + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (2 / 7) + 10, + ); const roadImages = extract_road_image_data(surveys[i], debug); + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (3 / 7) + 10, + ); + if (!(await this.service.mapMatch(surveys[i], roadImages))) { printJobError('Failed to map match road images.'); } + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (4 / 7) + 10, + ); const dashcameraImages = extract_dashcam_image_data(surveys[i], debug); + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (5 / 7) + 10, + ); + if (!(await this.service.mapMatch(surveys[i], dashcameraImages))) { printJobError('Failed to map match dashcam images.'); } - - await job.progress(65); + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (6 / 7) + 10, + ); // Upload all data and images to the database await Promise.all([ @@ -160,8 +180,12 @@ export class FileProcessor { ); }), ]); - await job.progress(99); + + await job.progress( + (95 - 10) * ((i + 1) / surveys.length) * (7 / 7) + 10, + ); } + await job.progress(95); // Delete the unzipped file try { diff --git a/backend/src/upload/upload.controller.ts b/backend/src/upload/upload.controller.ts index 25fde2ac..68250cdf 100644 --- a/backend/src/upload/upload.controller.ts +++ b/backend/src/upload/upload.controller.ts @@ -1,13 +1,13 @@ import { Body, Controller, + Get, HttpException, HttpStatus, InternalServerErrorException, Post, UploadedFile, UseInterceptors, - Get, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { InjectQueue } from '@nestjs/bull'; @@ -88,17 +88,14 @@ export class UploadController { 'completed', 'failed', ]); - const jobStatus = await Promise.all( + return await Promise.all( jobs.map(async (job) => ({ id: job.id, name: job.name, timestamp: job.timestamp, - jobData: job.data, status: await job.getState(), progress: await job.progress(), })), ); - console.log('Job Status:', jobStatus); - return jobStatus; } } diff --git a/frontend/src/Components/Conditions/UploadPanel.tsx b/frontend/src/Components/Conditions/UploadPanel.tsx index ec953b4c..4885d480 100644 --- a/frontend/src/Components/Conditions/UploadPanel.tsx +++ b/frontend/src/Components/Conditions/UploadPanel.tsx @@ -1,14 +1,18 @@ -import { FC, useRef, useState } from 'react'; +import { FC, useCallback, useEffect, useRef, useState } from 'react'; import SingleFileInput from '../Inputs/SingleFileInput'; import '../../css/upload_panel.css'; -import { uploadSurvey } from '../../queries/upload'; +import { getUploadStatus, uploadSurvey } from '../../queries/upload'; +import { UploadStatus } from '../../models/models'; interface Props { /** Event when user wants to close the panel */ close: () => void; } +/** Interval in seconds to refresh the upload status */ +const REFRESH_UPLOAD_STATUS_INTERVAL = 60; + /** * Create a panel where the user can upload a zip file * @@ -19,6 +23,49 @@ const UploadPanel: FC = ({ close }) => { 'waiting' | 'sending' | 'sent' | 'sent-error' | 'invalid-password' >('waiting'); + const [uploadStatus, setUploadStatus] = useState([]); + + const [waitingForUploadStatus, setWaitingForUploadStatus] = + useState(false); + + const actualizeUploadStatus = useCallback(() => { + getUploadStatus((status) => { + let nonActiveNumberToShow = 4; + status.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1)); + + setUploadStatus( + status.filter((s) => { + if (s.status === 'active') { + return true; + } + + if (nonActiveNumberToShow > 0) { + nonActiveNumberToShow--; + return true; + } + + return false; + }), + ); + }); + }, []); + + // First time + useEffect(() => { + if (actualizeUploadStatus) actualizeUploadStatus(); + }, []); + + // Refresh periodically + useEffect(() => { + if (actualizeUploadStatus) + setTimeout(() => { + if (waitingForUploadStatus) return; + setWaitingForUploadStatus(true); + actualizeUploadStatus(); + setWaitingForUploadStatus(false); + }, REFRESH_UPLOAD_STATUS_INTERVAL * 1000); + }, [uploadStatus]); + const passwordRef = useRef(null); return ( @@ -39,39 +86,89 @@ const UploadPanel: FC = ({ close }) => { - {state === 'waiting' && ( - { - setState('sending'); - - uploadSurvey( - file, - passwordRef.current?.value || '', - () => { - setState('sent'); - setTimeout(() => setState('waiting'), 3000); - }, - (error) => { - if (error.response.status === 403) { - setState('invalid-password'); - } else { - setState('sent-error'); - } - console.warn('Error while uploading new survey: ', error); - setTimeout(() => setState('waiting'), 3000); - }, - ); - }} - /> - )} - {state === 'sending' &&

Sending...

} - {state === 'sent' &&

Sent!

} - {state === 'sent-error' && ( -

Something went wrong while sending the file!

- )} - {state === 'invalid-password' &&

Invalid password!

} +
+ {state === 'waiting' && ( + { + setState('sending'); + + uploadSurvey( + file, + passwordRef.current?.value || '', + () => { + setState('sent'); + setTimeout(() => setState('waiting'), 3000); + }, + (error) => { + if (error.response.status === 403) { + setState('invalid-password'); + } else { + setState('sent-error'); + } + console.warn('Error while uploading new survey: ', error); + setTimeout(() => setState('waiting'), 3000); + }, + ); + }} + /> + )} + {state === 'sending' &&

Sending...

} + {state === 'sent' &&

Sent!

} + {state === 'sent-error' && ( +

Something went wrong while sending the file!

+ )} + {state === 'invalid-password' &&

Invalid password!

} +
+
+
+

Upload tasks status:

+ { + if (waitingForUploadStatus) return; + setWaitingForUploadStatus(true); + actualizeUploadStatus(); + setWaitingForUploadStatus(false); + }} + > + ↻ + +
+ {uploadStatus.length == 0 &&

No recent upload

} + {uploadStatus.length > 0 && ( + + + + + + + + + + + + {uploadStatus.map((status) => ( + + + + + + + + ))} + +
n°TypeDateStatusProgress
{status.id} + {status.name === 'process-file' + ? 'Data extraction' + : 'Unzip'} + + {new Date(status.timestamp).toLocaleDateString()}{' '} + {new Date(status.timestamp).toLocaleTimeString()} + {status.status}{Math.round(status.progress)}%
+ )} +
); diff --git a/frontend/src/css/single_file_input.css b/frontend/src/css/single_file_input.css index 41f87de0..a999e889 100644 --- a/frontend/src/css/single_file_input.css +++ b/frontend/src/css/single_file_input.css @@ -8,7 +8,7 @@ appearance: none; -webkit-appearance: none; border: #0076d3 2px solid; - padding: 10px 0 25px 0; + padding: 10px 0 30px 0; } .file p { diff --git a/frontend/src/css/upload_panel.css b/frontend/src/css/upload_panel.css index 4c02dd39..f2c543d8 100644 --- a/frontend/src/css/upload_panel.css +++ b/frontend/src/css/upload_panel.css @@ -17,10 +17,8 @@ display: flex; align-items: center; flex-direction: column; - height: 22%; - width: 20%; - min-width: 360px; - min-height: 200px; + width: 30%; + min-width: 550px; } .upload-panel-container h1 { @@ -30,7 +28,7 @@ .upload-panel-container h1, .upload-panel-container button { - margin-left: 25px; + margin-left: 110px; } .upload-panel-top button { background: none; @@ -39,7 +37,6 @@ width: 25px; height: 25px; color: white; - margin-left: 10px; } .upload-panel-top { @@ -61,8 +58,47 @@ border-radius: 5px; } +.upload-input-container { + padding: 0; + height: 89px; + width: 300px; +} + .upload-input { width: 300px; - height: 60px; - margin-bottom: 10px; + height: 50px; +} + +.upload-status { + margin: 15px 0 15px 0; +} + +.upload-status table { + width: 500px; +} + +.upload-status table tr td { + padding: 5px 0 5px 0; +} + +.upload-status-title-and-btn h4 { + margin: 10px 0 10px 0; + padding: 0; +} + +.upload-status-title-and-btn { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; +} + +.upload-status-title-and-btn span { + font-family: 'Lucida Sans Unicode', serif; + font-size: 30px; +} + +.upload-status-title-and-btn h4, +.upload-status-title-and-btn span { + margin-left: 50px; } diff --git a/frontend/src/models/models.ts b/frontend/src/models/models.ts index a8fc78cb..01ff803e 100644 --- a/frontend/src/models/models.ts +++ b/frontend/src/models/models.ts @@ -58,3 +58,12 @@ export interface SurveyListItem extends Path { timestamp: string; dynatest_id: number; } + +export interface UploadStatus { + id: number; + /** task name */ + name: string; + timestamp: number; + status: 'waiting' | 'active' | 'completed' | 'failed'; + progress: number; +} diff --git a/frontend/src/queries/upload.ts b/frontend/src/queries/upload.ts index ca4fc97a..6acb5d4e 100644 --- a/frontend/src/queries/upload.ts +++ b/frontend/src/queries/upload.ts @@ -1,4 +1,5 @@ -import { postForm } from './fetch'; +import { get, postForm } from './fetch'; +import { UploadStatus } from '../models/models'; /** * Uploads a survey to the backend. @@ -21,3 +22,7 @@ export const uploadSurvey = ( formData.append('password', password); postForm('/upload', formData, onSuccess, onError); }; + +export const getUploadStatus = (callback: (status: UploadStatus[]) => void) => { + get('/upload/status', callback); +};