From 878984d5dc365080dd88e2a12b148507ea98bc09 Mon Sep 17 00:00:00 2001 From: Sushant Date: Tue, 19 Nov 2024 16:45:23 +0530 Subject: [PATCH 1/3] 86c12bkmm - Climate mediator - Duplicate CSV Validation --- src/routes/index.ts | 20 ++++++++--- src/utils/minioClient.ts | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/routes/index.ts b/src/routes/index.ts index f9f5846..d1fdb4b 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -51,14 +51,24 @@ routes.post('/upload', upload.single('file'), async (req, res) => { return res.status(400).send('Invalid file type, please upload a valid CSV file'); } const fileUrl = saveCsvToTmp(file.buffer, file.originalname); - - const uploadResult = await uploadToMinio(fileUrl,file.originalname, bucket as string); - // const tableCreated = await createTable(headers, bucket as string); logger.info(`file created: ${file.originalname}`); - fs.unlinkSync(fileUrl); + try { + const uploadResult = await uploadToMinio(fileUrl, file.originalname, bucket as string); + // Clean up the temporary file + fs.unlinkSync(fileUrl); - return res.status(201).send('File uploaded successfully'); + if (uploadResult) { + return res.status(201).send(`File ${file.originalname} uploaded in bucket ${bucket}`); + } else { + return res.status(400).send(`Object ${file.originalname} already exists in bucket ${bucket}`); + } + } catch (error) { + // Clean up the temporary file in case of error + fs.unlinkSync(fileUrl); + logger.error('Error uploading file to Minio:', error); + return res.status(500).send('Error uploading file'); + } }); export default routes; diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts index e8507df..1e14afd 100644 --- a/src/utils/minioClient.ts +++ b/src/utils/minioClient.ts @@ -20,21 +20,74 @@ export async function uploadToMinio(sourceFile: string, destinationObject: strin accessKey, secretKey }); - // Check if bucket exists, create if it doesn't const exists = await minioClient.bucketExists(bucket); if (!exists) { await minioClient.makeBucket(bucket, bucketRegion); - logger.debug(`Bucket ${bucket} created in "${bucketRegion}".`); + logger.info(`Bucket ${bucket} created in "${bucketRegion}".`); } - // Set the object metadata - const metaData = { - 'Content-Type': 'text/plain', - ...customMetadata - }; - // Upload the file - await minioClient.fPutObject(bucket, destinationObject, sourceFile, metaData); - logger.debug(`File ${sourceFile} uploaded as object ${destinationObject} in bucket ${bucket}`); + try { + const fileExists = await checkCsvFileExists(destinationObject, bucket); + if (fileExists) { + return false; + } else { + const metaData = { + 'Content-Type': 'text/plain', + ...customMetadata + }; + + // Upload the file + await minioClient.fPutObject(bucket, destinationObject, sourceFile, metaData); + logger.info(`File ${sourceFile} uploaded as object ${destinationObject} in bucket ${bucket}`); + return true; + } + } catch (error) { + console.error('Error checking file:', error); + } +} + +/** + * Checks if a CSV file exists in the specified Minio bucket + * @param {string} fileName - Name of the CSV file to check + * @param {string} bucket - Bucket name + * @returns {Promise} - Returns true if file exists, false otherwise + */ +export async function checkCsvFileExists(fileName: string, bucket: string): Promise { + const minioClient = new Minio.Client({ + endPoint, + port, + useSSL, + accessKey, + secretKey + }); + + try { + // Check if bucket exists first + const bucketExists = await minioClient.bucketExists(bucket); + if (!bucketExists) { + logger.info(`Bucket ${bucket} does not exist`); + return false; + } + + // Get object stats to check if file exists + const stats = await minioClient.statObject(bucket, fileName); // Optionally verify it's a CSV file by checking Content-Type + if (stats.metaData && stats.metaData['content-type']) { + logger.info(`File ${fileName} exists in bucket ${bucket}`); + return true; + } else { + logger.info(`File ${fileName} does not exist in bucket ${bucket}`); + return false; + } + } catch (err: any) { + if (err.code === 'NotFound') { + logger.debug(`File ${fileName} not found in bucket ${bucket}`); + return false; + } + // For any other error, log it and rethrow + logger.error(`Error checking file existence: ${err.message}`); + throw err; + } + } From 31a047fec9920c553ffd9c131ffe03d8aa4e8ab1 Mon Sep 17 00:00:00 2001 From: Sushant Date: Tue, 19 Nov 2024 16:54:16 +0530 Subject: [PATCH 2/3] Added unique ID --- src/utils/minioClient.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts index 1e14afd..fe5bd7e 100644 --- a/src/utils/minioClient.ts +++ b/src/utils/minioClient.ts @@ -35,6 +35,7 @@ export async function uploadToMinio(sourceFile: string, destinationObject: strin } else { const metaData = { 'Content-Type': 'text/plain', + 'X-Upload-Id': crypto.randomUUID(), ...customMetadata }; From 8dda3c396e811950ec0a6b11f35a4b44fb7743cc Mon Sep 17 00:00:00 2001 From: Sushant Date: Tue, 19 Nov 2024 17:58:18 +0530 Subject: [PATCH 3/3] Added validation for CSV and JSON file type --- src/routes/index.ts | 67 ++++++++++++++++++++++++++++------------ src/utils/minioClient.ts | 10 +++--- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/routes/index.ts b/src/routes/index.ts index d1fdb4b..ea784c1 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -6,6 +6,7 @@ import logger from '../logger'; import fs from 'fs'; import path from 'path'; import { uploadToMinio } from '../utils/minioClient'; +import e from 'express'; const routes = express.Router(); const bodySizeLimit = getConfig().bodySizeLimit; @@ -31,6 +32,20 @@ const saveCsvToTmp = (fileBuffer: Buffer, fileName: string): string => { return fileUrl; }; +const isValidFileType = (file: Express.Multer.File): boolean => { + const validMimeTypes = ['text/csv', 'application/json']; + return validMimeTypes.includes(file.mimetype); +}; + +function validateJsonFile(buffer: Buffer): boolean { + try { + JSON.parse(buffer.toString()); + return true; + } catch { + return false; + } +} + routes.post('/upload', upload.single('file'), async (req, res) => { const file = req.file; const bucket = req.query.bucket; @@ -45,30 +60,44 @@ routes.post('/upload', upload.single('file'), async (req, res) => { return res.status(400).send('No bucket provided'); } - const headers = getCsvHeaders(file.buffer); - - if (!headers) { - return res.status(400).send('Invalid file type, please upload a valid CSV file'); + if (!isValidFileType(file)) { + logger.error(`Invalid file type: ${file.mimetype}`); + return res.status(400).send('Invalid file type. Please upload either a CSV or JSON file'); } - const fileUrl = saveCsvToTmp(file.buffer, file.originalname); - logger.info(`file created: ${file.originalname}`); - try { - const uploadResult = await uploadToMinio(fileUrl, file.originalname, bucket as string); - // Clean up the temporary file - fs.unlinkSync(fileUrl); + // For CSV files, validate headers + if (file.mimetype === 'text/csv') { + const headers = getCsvHeaders(file.buffer); + if (!headers) { + return res.status(400).send('Invalid CSV file format'); + } + try { + const fileUrl = saveCsvToTmp(file.buffer, file.originalname); + const uploadResult = await uploadToMinio(fileUrl, file.originalname, bucket as string, file.mimetype); + // Clean up the temporary file + fs.unlinkSync(fileUrl); - if (uploadResult) { - return res.status(201).send(`File ${file.originalname} uploaded in bucket ${bucket}`); - } else { - return res.status(400).send(`Object ${file.originalname} already exists in bucket ${bucket}`); + if (uploadResult) { + return res.status(201).send(`File ${file.originalname} uploaded in bucket ${bucket}`); + } else { + return res.status(400).send(`Object ${file.originalname} already exists in bucket ${bucket}`); + } + } catch (error) { + // Clean up the temporary file in case of error + fs.unlinkSync(fileUrl); + logger.error('Error uploading file to Minio:', error); + return res.status(500).send('Error uploading file'); + } + } else if (file.mimetype === 'application/json') { + if (!validateJsonFile(file.buffer)) { + return res.status(400).send('Invalid JSON file format'); } - } catch (error) { - // Clean up the temporary file in case of error - fs.unlinkSync(fileUrl); - logger.error('Error uploading file to Minio:', error); - return res.status(500).send('Error uploading file'); + + return res.status(200).send('JSON file is valid - Future implementation'); + } else { + return res.status(400).send('Invalid file type. Please upload either a CSV or JSON file'); } + }); export default routes; diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts index fe5bd7e..d3ade97 100644 --- a/src/utils/minioClient.ts +++ b/src/utils/minioClient.ts @@ -12,7 +12,7 @@ const {endPoint, port, useSSL, bucketRegion, accessKey, secretKey, prefix, suffi * @param {Object} [customMetadata={}] - Optional custom metadata * @returns {Promise} */ -export async function uploadToMinio(sourceFile: string, destinationObject: string, bucket: string, customMetadata = {}) { +export async function uploadToMinio(sourceFile: string, destinationObject: string, bucket: string, fileType: string, customMetadata = {}) { const minioClient = new Minio.Client({ endPoint, port, @@ -29,12 +29,12 @@ export async function uploadToMinio(sourceFile: string, destinationObject: strin try { - const fileExists = await checkCsvFileExists(destinationObject, bucket); + const fileExists = await checkFileExists(destinationObject, bucket, fileType); if (fileExists) { return false; } else { const metaData = { - 'Content-Type': 'text/plain', + 'Content-Type': fileType, 'X-Upload-Id': crypto.randomUUID(), ...customMetadata }; @@ -55,7 +55,7 @@ export async function uploadToMinio(sourceFile: string, destinationObject: strin * @param {string} bucket - Bucket name * @returns {Promise} - Returns true if file exists, false otherwise */ -export async function checkCsvFileExists(fileName: string, bucket: string): Promise { +export async function checkFileExists(fileName: string, bucket: string, fileType: string): Promise { const minioClient = new Minio.Client({ endPoint, port, @@ -74,7 +74,7 @@ export async function checkCsvFileExists(fileName: string, bucket: string): Prom // Get object stats to check if file exists const stats = await minioClient.statObject(bucket, fileName); // Optionally verify it's a CSV file by checking Content-Type - if (stats.metaData && stats.metaData['content-type']) { + if (stats.metaData && stats.metaData['content-type'] === fileType) { logger.info(`File ${fileName} exists in bucket ${bucket}`); return true; } else {