fix: fixes stringifying of large asset maps by streaming the JSON to …

…file.

package.json

@@ -41,7 +41,8 @@
     "rimraf": "^6.0.1",
     "split2": "^4.2.0",
     "tar": "^7.0.1",
-    "yaml": "^2.4.2"
+    "yaml": "^2.4.2",
+    "json-stream-stringify": "^2.0.2"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

src/export.js

@@ -5,8 +5,10 @@ const zlib = require('zlib')
 const archiver = require('archiver')
 const miss = require('mississippi')
 const split = require('split2')
+const JsonStreamStringify = require('json-stream-stringify')
 const AssetHandler = require('./AssetHandler')
 const debug = require('./debug')
+const pipeAsync = require('./util/pipeAsync')
 const filterDocumentTypes = require('./filterDocumentTypes')
 const filterDrafts = require('./filterDrafts')
 const filterSystemDocuments = require('./filterSystemDocuments')
@@ -43,6 +45,7 @@ async function exportDataset(opts) {
   const tmpDir = path.join(os.tmpdir(), prefix)
   fs.mkdirSync(tmpDir, {recursive: true})
   const dataPath = path.join(tmpDir, 'data.ndjson')
+  const assetsPath = path.join(tmpDir, 'assets.json')
 
   const cleanup = () =>
     rimraf(tmpDir).catch((err) => {
@@ -209,7 +212,9 @@ async function exportDataset(opts) {
         update: true,
       })
 
-      archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix})
+      const assetsStream = fs.createWriteStream(assetsPath)
+      await pipeAsync(new JsonStreamStringify(assetMap), assetsStream)
+      archive.file(assetsPath, {name: 'assets.json', prefix})
       clearInterval(progressInterval)
     } catch (assetErr) {
       clearInterval(progressInterval)

src/util/pipeAsync.js

@@ -0,0 +1,17 @@
+const miss = require('mississippi')
+
+module.exports = async (readable, writable) => {
+  return new Promise((resolve, reject) => {
+    try {
+      miss.pipe(readable, writable, (jsonErr) => {
+        if (jsonErr) {
+          reject(jsonErr)
+        } else {
+          resolve()
+        }
+      })
+    } catch (assetErr) {
+      reject(assetErr)
+    }
+  })
+}