Show upload status on the frontend

backend/src/upload/file.processor.ts

@@ -46,8 +46,9 @@ export class FileProcessor {
           console.error(`Error unzipping file: ${filePath}`, error);
         })
         .on('close', async () => {
-          await this.handleUnzippedContent(tempUnzipPath, filePath);
           await job.progress(60);
+          await this.handleUnzippedContent(tempUnzipPath, filePath);
+          await job.progress(100);
         });
     } catch (error) {
       console.error(`Error processing file: ${job.data.filePath}`, error);
@@ -103,6 +104,7 @@ export class FileProcessor {
 
       //here we make sure that there is at least one RSP file and a HDC directory
       let surveys = find_surveys(filePath, debug);
+      await job.progress(10);
       if (debug) {
         printJobInfo(surveys);
       }
@@ -111,7 +113,6 @@ export class FileProcessor {
         printJobInfo('No valid data found in directory: ' + filePath);
       }
 
-      // TODO: split process here instead of for the all zip file (one process per survey)
       for (let i = 0; i < surveys.length; i++) {
         // upload the survey data and get the id back
         surveys[i].fk_survey_id = await this.service.db_insert_survey_data(
@@ -120,21 +121,40 @@ export class FileProcessor {
         );
 
         const data = extract_measurements_data(surveys[i], debug);
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (1 / 7) + 10,
+        );
+
         if (!(await this.service.mapMatch(surveys[i], data))) {
           printJobError('Failed to map match data.');
         }
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (2 / 7) + 10,
+        );
 
         const roadImages = extract_road_image_data(surveys[i], debug);
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (3 / 7) + 10,
+        );
+
         if (!(await this.service.mapMatch(surveys[i], roadImages))) {
           printJobError('Failed to map match road images.');
         }
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (4 / 7) + 10,
+        );
 
         const dashcameraImages = extract_dashcam_image_data(surveys[i], debug);
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (5 / 7) + 10,
+        );
+
         if (!(await this.service.mapMatch(surveys[i], dashcameraImages))) {
           printJobError('Failed to map match dashcam images.');
         }
-
-        await job.progress(65);
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (6 / 7) + 10,
+        );
 
         // Upload all data and images to the database
         await Promise.all([
@@ -160,8 +180,12 @@ export class FileProcessor {
             );
           }),
         ]);
-        await job.progress(99);
+
+        await job.progress(
+          (95 - 10) * ((i + 1) / surveys.length) * (7 / 7) + 10,
+        );
       }
+      await job.progress(95);
 
       // Delete the unzipped file
       try {

backend/src/upload/upload.controller.ts

@@ -1,13 +1,13 @@
 import {
   Body,
   Controller,
+  Get,
   HttpException,
   HttpStatus,
   InternalServerErrorException,
   Post,
   UploadedFile,
   UseInterceptors,
-  Get,
 } from '@nestjs/common';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { InjectQueue } from '@nestjs/bull';
@@ -88,17 +88,14 @@ export class UploadController {
       'completed',
       'failed',
     ]);
-    const jobStatus = await Promise.all(
+    return await Promise.all(
       jobs.map(async (job) => ({
         id: job.id,
         name: job.name,
         timestamp: job.timestamp,
-        jobData: job.data,
         status: await job.getState(),
         progress: await job.progress(),
       })),
     );
-    console.log('Job Status:', jobStatus);
-    return jobStatus;
   }
 }

frontend/src/Components/Conditions/UploadPanel.tsx

@@ -1,14 +1,18 @@
-import { FC, useRef, useState } from 'react';
+import { FC, useCallback, useEffect, useRef, useState } from 'react';
 import SingleFileInput from '../Inputs/SingleFileInput';
 
 import '../../css/upload_panel.css';
-import { uploadSurvey } from '../../queries/upload';
+import { getUploadStatus, uploadSurvey } from '../../queries/upload';
+import { UploadStatus } from '../../models/models';
 
 interface Props {
   /** Event when user wants to close the panel */
   close: () => void;
 }
 
+/** Interval in seconds to refresh the upload status */
+const REFRESH_UPLOAD_STATUS_INTERVAL = 60;
+
 /**
  * Create a panel where the user can upload a zip file
  *
@@ -19,6 +23,49 @@ const UploadPanel: FC<Props> = ({ close }) => {
     'waiting' | 'sending' | 'sent' | 'sent-error' | 'invalid-password'
   >('waiting');
 
+  const [uploadStatus, setUploadStatus] = useState<UploadStatus[]>([]);
+
+  const [waitingForUploadStatus, setWaitingForUploadStatus] =
+    useState<boolean>(false);
+
+  const actualizeUploadStatus = useCallback(() => {
+    getUploadStatus((status) => {
+      let nonActiveNumberToShow = 4;
+      status.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
+
+      setUploadStatus(
+        status.filter((s) => {
+          if (s.status === 'active') {
+            return true;
+          }
+
+          if (nonActiveNumberToShow > 0) {
+            nonActiveNumberToShow--;
+            return true;
+          }
+
+          return false;
+        }),
+      );
+    });
+  }, []);
+
+  // First time
+  useEffect(() => {
+    if (actualizeUploadStatus) actualizeUploadStatus();
+  }, []);
+
+  // Refresh periodically
+  useEffect(() => {
+    if (actualizeUploadStatus)
+      setTimeout(() => {
+        if (waitingForUploadStatus) return;
+        setWaitingForUploadStatus(true);
+        actualizeUploadStatus();
+        setWaitingForUploadStatus(false);
+      }, REFRESH_UPLOAD_STATUS_INTERVAL * 1000);
+  }, [uploadStatus]);
+
   const passwordRef = useRef<HTMLInputElement>(null);
 
   return (
@@ -39,39 +86,89 @@ const UploadPanel: FC<Props> = ({ close }) => {
           <input ref={passwordRef} name="password" type="password" />
         </div>
 
-        {state === 'waiting' && (
-          <SingleFileInput
-            className="upload-input"
-            displayName="Upload a .zip"
-            onFileDrop={(file: File) => {
-              setState('sending');
-
-              uploadSurvey(
-                file,
-                passwordRef.current?.value || '',
-                () => {
-                  setState('sent');
-                  setTimeout(() => setState('waiting'), 3000);
-                },
-                (error) => {
-                  if (error.response.status === 403) {
-                    setState('invalid-password');
-                  } else {
-                    setState('sent-error');
-                  }
-                  console.warn('Error while uploading new survey: ', error);
-                  setTimeout(() => setState('waiting'), 3000);
-                },
-              );
-            }}
-          />
-        )}
-        {state === 'sending' && <p>Sending...</p>}
-        {state === 'sent' && <p>Sent!</p>}
-        {state === 'sent-error' && (
-          <p>Something went wrong while sending the file!</p>
-        )}
-        {state === 'invalid-password' && <p>Invalid password!</p>}
+        <div className="upload-input-container">
+          {state === 'waiting' && (
+            <SingleFileInput
+              className="upload-input"
+              displayName="Upload a .zip"
+              onFileDrop={(file: File) => {
+                setState('sending');
+
+                uploadSurvey(
+                  file,
+                  passwordRef.current?.value || '',
+                  () => {
+                    setState('sent');
+                    setTimeout(() => setState('waiting'), 3000);
+                  },
+                  (error) => {
+                    if (error.response.status === 403) {
+                      setState('invalid-password');
+                    } else {
+                      setState('sent-error');
+                    }
+                    console.warn('Error while uploading new survey: ', error);
+                    setTimeout(() => setState('waiting'), 3000);
+                  },
+                );
+              }}
+            />
+          )}
+          {state === 'sending' && <p>Sending...</p>}
+          {state === 'sent' && <p>Sent!</p>}
+          {state === 'sent-error' && (
+            <p>Something went wrong while sending the file!</p>
+          )}
+          {state === 'invalid-password' && <p>Invalid password!</p>}
+        </div>
+        <div className="upload-status">
+          <div className="upload-status-title-and-btn">
+            <h4>Upload tasks status:</h4>
+            <span
+              title="Manually refresh the list"
+              onClick={() => {
+                if (waitingForUploadStatus) return;
+                setWaitingForUploadStatus(true);
+                actualizeUploadStatus();
+                setWaitingForUploadStatus(false);
+              }}
+            >
+              &#x21bb;
+            </span>
+          </div>
+          {uploadStatus.length == 0 && <p>No recent upload</p>}
+          {uploadStatus.length > 0 && (
+            <table>
+              <thead>
+                <tr>
+                  <th>n°</th>
+                  <th>Type</th>
+                  <th>Date</th>
+                  <th>Status</th>
+                  <th>Progress</th>
+                </tr>
+              </thead>
+              <tbody>
+                {uploadStatus.map((status) => (
+                  <tr key={status.id}>
+                    <td>{status.id}</td>
+                    <td>
+                      {status.name === 'process-file'
+                        ? 'Data extraction'
+                        : 'Unzip'}
+                    </td>
+                    <td>
+                      {new Date(status.timestamp).toLocaleDateString()}{' '}
+                      {new Date(status.timestamp).toLocaleTimeString()}
+                    </td>
+                    <td>{status.status}</td>
+                    <td>{Math.round(status.progress)}%</td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          )}
+        </div>
       </div>
     </div>
   );

frontend/src/css/single_file_input.css

@@ -8,7 +8,7 @@
   appearance: none;
   -webkit-appearance: none;
   border: #0076d3 2px solid;
-  padding: 10px 0 25px 0;
+  padding: 10px 0 30px 0;
 }
 
 .file p {