diff --git a/server/src/jobs/partenaireExport/exportJobsToS3.ts b/server/src/jobs/partenaireExport/exportJobsToS3.ts index ad97b04c40..7d7a92c7f7 100644 --- a/server/src/jobs/partenaireExport/exportJobsToS3.ts +++ b/server/src/jobs/partenaireExport/exportJobsToS3.ts @@ -1,7 +1,10 @@ -import { writeFileSync } from "fs" +import { createWriteStream } from "fs" +import { Transform } from "stream" +import { pipeline } from "stream/promises" import { LBA_ITEM_TYPE } from "shared/constants/lbaitem" +import { logger } from "../../common/logger" import { uploadFileToS3 } from "../../common/utils/awsUtils" import { getDbCollection } from "../../common/utils/mongodbUtils" @@ -13,9 +16,35 @@ interface IGeneratorParams { } async function generateJsonExport({ collection, query, projection, fileName }: IGeneratorParams): Promise { + logger.info(`Generating file ${fileName}`) const filePath = new URL(`./${fileName}.json`, import.meta.url) - const data = await getDbCollection(collection).find(query).project(projection).toArray() - writeFileSync(filePath, JSON.stringify(data, null, 4)) + const cursor = await getDbCollection(collection).find(query).project(projection) + const writable = createWriteStream(filePath) + + // Transform stream to add commas between objects and brackets at the beginning and end + let isFirst = true + const transform = new Transform({ + writableObjectMode: true, + readableObjectMode: true, + transform(chunk, encoding, callback) { + if (isFirst) { + this.push("[") + this.push(JSON.stringify(chunk, null, 4)) + isFirst = false + } else { + this.push(",") + this.push(JSON.stringify(chunk, null, 4)) + } + callback() + }, + flush(callback) { + this.push("]") + callback() + }, + }) + + await pipeline(cursor, transform, writable) + return filePath.pathname } @@ -54,8 +83,18 @@ async function exportLbaJobsToS3() { fileName: LBA_ITEM_TYPE.RECRUTEURS_LBA, } await Promise.all([ - generateJsonExport(offres_emploi_lba).then((path) => uploadFileToS3({ key: path.split("/").pop() as string, filePath: path, noCache: true })), - generateJsonExport(recruteurs_lba).then((path) => uploadFileToS3({ key: path.split("/").pop() as string, filePath: path, noCache: true })), + generateJsonExport(offres_emploi_lba).then((path) => { + const key = path.split("/").pop() as string + logger.info(`Uploading file ${key} to S3`) + uploadFileToS3({ key, filePath: path, noCache: true }) + logger.info(`file ${key} uploaded`) + }), + generateJsonExport(recruteurs_lba).then((path) => { + const key = path.split("/").pop() as string + logger.info(`Uploading file ${key} to S3`) + uploadFileToS3({ key, filePath: path, noCache: true }) + logger.info(`File ${key} uploaded`) + }), ]) }