diff --git a/package-lock.json b/package-lock.json index 0a0971e..33d763e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "archiver": "^7.0.0", "debug": "^4.3.4", "get-it": "^8.6.2", + "json-stream-stringify": "^2.0.2", "lodash": "^4.17.21", "mississippi": "^4.0.0", "p-queue": "^2.3.0", @@ -6774,6 +6775,11 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/json-stream-stringify": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/json-stream-stringify/-/json-stream-stringify-2.0.4.tgz", + "integrity": "sha512-gIPoa6K5w6j/RnQ3fOtmvICKNJGViI83A7dnTIL+0QJ/1GKuNvCPFvbFWxt0agruF4iGgDFJvge4Gua4ZoiggQ==" + }, "node_modules/json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", diff --git a/package.json b/package.json index ba2bd2c..9aed557 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "rimraf": "^6.0.1", "split2": "^4.2.0", "tar": "^7.0.1", - "yaml": "^2.4.2" + "yaml": "^2.4.2", + "json-stream-stringify": "^2.0.2" }, "devDependencies": { "@jest/globals": "^29.7.0", diff --git a/src/export.js b/src/export.js index e4d8072..dcd83e3 100644 --- a/src/export.js +++ b/src/export.js @@ -5,8 +5,10 @@ const zlib = require('zlib') const archiver = require('archiver') const miss = require('mississippi') const split = require('split2') +const JsonStreamStringify = require('json-stream-stringify') const AssetHandler = require('./AssetHandler') const debug = require('./debug') +const pipeAsync = require('./util/pipeAsync') const filterDocumentTypes = require('./filterDocumentTypes') const filterDrafts = require('./filterDrafts') const filterSystemDocuments = require('./filterSystemDocuments') @@ -43,6 +45,7 @@ async function exportDataset(opts) { const tmpDir = path.join(os.tmpdir(), prefix) fs.mkdirSync(tmpDir, {recursive: true}) const dataPath = path.join(tmpDir, 'data.ndjson') + const assetsPath = path.join(tmpDir, 'assets.json') const cleanup = () => rimraf(tmpDir).catch((err) => { @@ -209,7 +212,9 @@ async function exportDataset(opts) { update: true, }) - archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix}) + const assetsStream = fs.createWriteStream(assetsPath) + await pipeAsync(new JsonStreamStringify(assetMap), assetsStream) + archive.file(assetsPath, {name: 'assets.json', prefix}) clearInterval(progressInterval) } catch (assetErr) { clearInterval(progressInterval) diff --git a/src/util/pipeAsync.js b/src/util/pipeAsync.js new file mode 100644 index 0000000..81dd117 --- /dev/null +++ b/src/util/pipeAsync.js @@ -0,0 +1,17 @@ +const miss = require('mississippi') + +module.exports = async (readable, writable) => { + return new Promise((resolve, reject) => { + try { + miss.pipe(readable, writable, (jsonErr) => { + if (jsonErr) { + reject(jsonErr) + } else { + resolve() + } + }) + } catch (assetErr) { + reject(assetErr) + } + }) +}