Skip to content

Commit

Permalink
fileSystem: fix large zip files extraction
Browse files Browse the repository at this point in the history
fileSystem.extract wasn't able to extract large
zip archives (> 2Go).
adm-zip package doesn't support extracting huge
archives because it loads the whole archive into
memory.

adm-zip package has been replaced by
node-stream-zip.
  • Loading branch information
maxime-beguin committed Apr 4, 2022
1 parent ea49294 commit 14ff726
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 18 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# 8.0.4 / YYYY-MM-DD

## BUG FIXES

- Fix require('@openveo/api').fileSystem.extract which wasn't able to extract large zip files (more than 2Go)

## DEPENDENCIES

- **adm-zip** has been replaced by node-stream-zip

# 8.0.3 / 2022-03-10

## BUG FIXES
Expand Down
17 changes: 14 additions & 3 deletions lib/fileSystem.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ var path = require('path');
var stream = require('stream');
var util = require('util');

var AdmZip = require('adm-zip');
var async = require('async');
var nanoid = require('nanoid').nanoid;
var StreamZip = require('node-stream-zip');
var tar = require('tar-fs');

/**
Expand Down Expand Up @@ -68,8 +68,14 @@ function extractTarFile(filePath, destinationPath, callback) {
* @param {callback} [callback] The function to call when done
*/
function extractZipFile(filePath, destinationPath, callback) {
var zip = new AdmZip(path.normalize(filePath));
zip.extractAllToAsync(path.normalize(destinationPath), true, true, callback);
var zip = new StreamZip({file: filePath});
zip.on('error', callback);
zip.on('ready', function() {
zip.extract(null, destinationPath, function(error, count) {
zip.close();
callback(error);
});
});
}

/**
Expand Down Expand Up @@ -613,6 +619,11 @@ module.exports.extract = function(filePath, destinationPath, callback) {

async.waterfall([

// Make sure destination directory exists
function(callback) {
self.mkdir(destinationPath, callback);
},

// Get file type
function(callback) {
self.getFileType(filePath, callback);
Expand Down
32 changes: 18 additions & 14 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
"url": "https://github.com/veo-labs/openveo-api.git"
},
"dependencies": {
"adm-zip": "^0.5.9",
"async": "^3.2.0",
"connect-mongo": "^4.4.1",
"esprima": "^4.0.1",
Expand All @@ -31,6 +30,7 @@
"mongodb": "^3.6.10",
"multer": "^1.4.3",
"nanoid": "^3.1.23",
"node-stream-zip": "^1.15.0",
"nopt": "^5.0.0",
"passport": "^0.4.1",
"passport-ldapauth": "^3.0.1",
Expand Down

0 comments on commit 14ff726

Please sign in to comment.