From 9bc4657b4497e8c69d5e1279fafd6edecf1a8498 Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Fri, 29 May 2015 14:05:12 -0700 Subject: [PATCH 1/6] Call callback when operations are done, closes db, and add only, except support --- index.js | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index c4cd58d..020cf6d 100644 --- a/index.js +++ b/index.js @@ -13,7 +13,7 @@ var reader = require('read-async-bson'); module.exports = { // If you leave out "stream" it'll be stdout - dump: function(dbOrUri, stream, callback) { + dump: function(dbOrUri, stream, callback, opt) { if (arguments.length === 2) { callback = stream; stream = undefined; @@ -21,6 +21,7 @@ module.exports = { if (!stream) { stream = process.stdout; } + var opt = opt || {}; var db; var out = stream; var endOfCollection = crypto.pseudoRandomBytes(8).toString('base64'); @@ -50,6 +51,14 @@ module.exports = { return callback(err); } collections = _collections; + if (opt.only) + { + collections = _.filter(collections, function(collection){ return opt.only.indexOf(collection.collectionName) != -1; }); + } + if (opt.except) + { + collections = _.filter(collections, function(collection){ return opt.except.indexOf(collection.collectionName) == -1; }); + } return callback(null); }); }, @@ -106,10 +115,18 @@ module.exports = { }, callback); }, endDatabase: function(callback) { + out.once('drain', function() + { + setImmediate(callback); + }) write({ type: 'endDatabase' }); - return setImmediate(callback); + }, + disconnect: function(callback) { + db.close(); + db = null; + callback(null); } }, function(err) { if (err) { @@ -216,7 +233,6 @@ module.exports = { } return collection.insert(item.document, callback); } - // Just scan for the unique random string that // appears in the end-of-collection object. No need // to waste time parsing BSON for this. Also @@ -310,6 +326,11 @@ module.exports = { } return finalCallback(new Error('Premature end of stream')); }); + }, + disconnect: function(callback){ + db.close(); + db = null; + callback(null); } }, callback); } From 9c87c62c643c176b3c1f6df0f3d97d7aba1805b5 Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Thu, 25 Feb 2016 16:30:26 -0800 Subject: [PATCH 2/6] Change from cursor.nextObject to cursor.each --- index.js | 77 ++++++++++++++++++++++++++++++++++------------------ package.json | 2 +- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/index.js b/index.js index 020cf6d..49ae5a4 100644 --- a/index.js +++ b/index.js @@ -11,6 +11,7 @@ var buffertools = require('buffertools'); var crypto = require('crypto'); var reader = require('read-async-bson'); + module.exports = { // If you leave out "stream" it'll be stdout dump: function(dbOrUri, stream, callback, opt) { @@ -83,32 +84,56 @@ module.exports = { }, getDocuments: function(callback) { var cursor = collection.find({}, { raw: true }); - iterate(); - function iterate() { - return cursor.nextObject(function(err, item) { - if (err) { - return callback(err); - } - if (!item) { - write({ - // Ensures we don't confuse this with - // a legitimate database object - endOfCollection: endOfCollection - }); - return callback(null); - } + cursor.each(function (err, doc) { + if (err) { + return callback(err); + } + + if (err) { + return callback(err); + } else if (doc) { // v2: just a series of actual data documents - out.write(item); - - // If we didn't have the raw BSON document, - // we could do this instead, but it would be very slow - // write({ - // type: 'document', - // document: item - // }); - - return setImmediate(iterate); - }); + out.write(doc); + } else { + write({ + // Ensures we don't confuse this with + // a legitimate database object + endOfCollection: endOfCollection + }); + cursor.close(); + return callback(null); + } + + // v2: just a series of actual data documents + out.write(doc); + }); + + // iterate(); +// function iterate() { +// return cursor.nextObject(function(err, item) { +// if (err) { +// return callback(err); +// } +// if (!item) { +// write({ +// // Ensures we don't confuse this with +// // a legitimate database object +// endOfCollection: endOfCollection +// }); +// return callback(null); +// } +// // v2: just a series of actual data documents +// out.write(item); +// +// // If we didn't have the raw BSON document, +// // we could do this instead, but it would be very slow +// // write({ +// // type: 'document', +// // document: item +// // }); +// +// return setImmediate(iterate); +// }); } }, }, callback); @@ -251,7 +276,7 @@ module.exports = { } return async.series({ flushIfNeeded: function(callback) { - if (insertQueueSize + item.length <= 16777216) { + if (insertQueueSize + item.length <= 16770000){ //16777216) { return setImmediate(callback); } return flush(callback); diff --git a/package.json b/package.json index 32491f5..5a6474d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mongo-dump-stream", - "version": "0.3.2", + "version": "0.3.3", "description": "Dump and restore an entire MongoDB database via a single stream, without creating a folder of files", "main": "index.js", "scripts": { From 3c4f6c7f3cd52b7120a80a957ccf8f70671279d5 Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Thu, 25 Feb 2016 16:41:08 -0800 Subject: [PATCH 3/6] Fix bad commenting --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 49ae5a4..cc28f8d 100644 --- a/index.js +++ b/index.js @@ -134,7 +134,7 @@ module.exports = { // // return setImmediate(iterate); // }); - } +// } }, }, callback); }, callback); From ca2ba3b1d9f42bf3214b1cfbccd9bc1235831577 Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Thu, 25 Feb 2016 17:59:54 -0800 Subject: [PATCH 4/6] Use pipe instead --- index.js | 55 +++++++------------------------------------------------ 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/index.js b/index.js index cc28f8d..ac5e4ee 100644 --- a/index.js +++ b/index.js @@ -83,70 +83,29 @@ module.exports = { }); }, getDocuments: function(callback) { - var cursor = collection.find({}, { raw: true }); - cursor.each(function (err, doc) { - if (err) { - return callback(err); - } - - if (err) { - return callback(err); - } else if (doc) { - // v2: just a series of actual data documents - out.write(doc); - } else { + var cursor = collection.find({}, { raw: true }).stream(); + cursor.pipe(out, { end: false }); + cursor.on('end', function() { write({ // Ensures we don't confuse this with // a legitimate database object endOfCollection: endOfCollection }); - cursor.close(); return callback(null); - } - - // v2: just a series of actual data documents - out.write(doc); }); - // iterate(); -// function iterate() { -// return cursor.nextObject(function(err, item) { -// if (err) { -// return callback(err); -// } -// if (!item) { -// write({ -// // Ensures we don't confuse this with -// // a legitimate database object -// endOfCollection: endOfCollection -// }); -// return callback(null); -// } -// // v2: just a series of actual data documents -// out.write(item); -// -// // If we didn't have the raw BSON document, -// // we could do this instead, but it would be very slow -// // write({ -// // type: 'document', -// // document: item -// // }); -// -// return setImmediate(iterate); -// }); -// } + cursor.on('error', function(err) { + return callback(err); + }); }, }, callback); }, callback); }, endDatabase: function(callback) { - out.once('drain', function() - { - setImmediate(callback); - }) write({ type: 'endDatabase' }); + callback(null); }, disconnect: function(callback) { db.close(); From cff914e592fc7fc9c6f3432d15c2b36399cb398c Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Wed, 15 Mar 2017 16:16:15 -0700 Subject: [PATCH 5/6] Bump to newer version of mongodb --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5a6474d..a54c77e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "bson": "^0.2.19", "buffertools": "^2.1.2", "lodash": "^3.3.1", - "mongodb": "^1.4.32", + "mongodb": "^2.2.19", "yargs": "^3.4.4", "read-async-bson": "^0.1.0" }, From 354b08bfc9109367ff30deb8ecf9b79e7fcdece0 Mon Sep 17 00:00:00 2001 From: Jack Yeh Date: Wed, 15 Mar 2017 16:17:12 -0700 Subject: [PATCH 6/6] 0.3.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a54c77e..d8a87b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mongo-dump-stream", - "version": "0.3.3", + "version": "0.3.4", "description": "Dump and restore an entire MongoDB database via a single stream, without creating a folder of files", "main": "index.js", "scripts": {