From 7d19dc7d3bd8360235334650da2bf73a197377c7 Mon Sep 17 00:00:00 2001 From: Vitaliy Semenchenko Date: Mon, 13 May 2019 11:04:38 +0300 Subject: [PATCH] Replace callbacks with promises --- lib/filestorage.js | 129 ++++------ lib/fs.js | 30 +++ lib/utils.js | 114 ++++----- package-lock.json | 148 ++++++------ package.json | 10 +- test/filestorage.js | 575 +++++++++++++++----------------------------- test/utils.js | 47 ++-- 7 files changed, 425 insertions(+), 628 deletions(-) create mode 100644 lib/fs.js diff --git a/lib/filestorage.js b/lib/filestorage.js index e8960a7..78fa235 100644 --- a/lib/filestorage.js +++ b/lib/filestorage.js @@ -1,6 +1,6 @@ 'use strict'; -const fs = require('fs'); +const fs = require('./fs'); const path = require('path'); const common = require('@metarhia/common'); const utils = require('./utils'); @@ -12,126 +12,95 @@ const getFilepath = Symbol('getFilepath'); class FileStorage { // Create new FileStorage // options - - // dir - , data storage directory, which should be created + // path - , data storage directory, which should be created // before FileStorage is used // minCompressSize - , minimal file size // to be compressed, default = 1024 constructor(options) { - this.dir = path.resolve(options.dir); + this.path = path.resolve(options.path); this.minCompressSize = options.minCompressSize || MIN_COMPRESS_SIZE; } // Write file to storage - // id - , id of file + // id - | | , id of file // data - | | , data to be written // opts - // checksum - , checksum type // dedupHash - , second checksum type - // cb - , callback - // err - - // stats - - // checksum - , data checksum - // dedupHash - , second data checksum - // size - , data size + // Returns: , stats + // checksum - , data checksum + // dedupHash - , second data checksum + // size - , data size // Throws: , if `opts.checksum` or `opts.dedupHash` is incorrect - write(id, data, opts, cb) { + async write(id, data, opts) { const file = this[getFilepath](id); - common.mkdirp(path.dirname(file), err => { - if (err) { - cb(err); - return; - } - fs.writeFile(file, data, err => { - if (err) { - cb(err); - return; - } - const stats = utils.getDataStats(data, opts.checksum, opts.dedupHash); - cb(null, stats); - }); - }); + await utils.mkdirRecursive(path.dirname(file)); + await fs.writeFile(file, data); + + return utils.getDataStats(data, opts.checksum, opts.dedupHash); } // Update file in the storage - // id - , id of file + // id - | | , id of file // data - | | , data to be written // opts - // checksum - , checksum type // dedupHash - , second checksum type - // cb - , callback - // err - - // stats - - // checksum - , data checksum - // dedupHash - , second data checksum - // size - , data size - // originalSize - , size of original file + // Returns: , stats + // checksum - , data checksum + // dedupHash - , second data checksum + // size - , data size + // originalSize - , size of original file // Throws: , if `opts.checksum` or `opts.dedupHash` is incorrect - update(id, data, opts, cb) { + async update(id, data, opts) { const file = this[getFilepath](id); - fs.stat(file, (err, fstats) => { - if (err) { - cb(err); - return; - } - fs.writeFile(file, data, err => { - if (err) { - cb(err); - return; - } - const stats = utils.getDataStats(data, opts.checksum, opts.dedupHash); - stats.originalSize = fstats.size; - cb(null, stats); - }); - }); + const fstats = await fs.stat(file); + await fs.writeFile(file, data); + + const stats = utils.getDataStats(data, opts.checksum, opts.dedupHash); + stats.originalSize = fstats.size; + return stats; } // Get information about file - // id - , id of file - // cb - , callback - // err - - // stats - - stat(id, cb) { + // id - | | , id of file + // Returns: + async stat(id) { const file = this[getFilepath](id); - fs.stat(file, cb); + return fs.stat(file); } // Read file from storage - // id - , id of file + // id - | | , id of file // opts - // encoding - // compression - - // cb - , callback - // err - - // data - | - read(id, opts, cb) { + // Returns: | , data + async read(id, opts) { const file = this[getFilepath](id); - if (opts.compression) utils.uncompress(file, opts, cb); - else fs.readFile(file, opts.encoding, cb); + if (opts.compression) return utils.uncompress(file, opts); + return fs.readFile(file, opts.encoding); } // Delete file from storage - // id - , id of file - // cb - , callback - // err - - rm(id, cb) { + // id - | | , id of file + async rm(id) { const file = this[getFilepath](id); - fs.unlink(file, cb); + await fs.unlink(file); } // Compress file in storage - // id - , id of file + // id - | | , id of file // compression - , compression type - // cb - , callback - // err - - // compressed - , whether file was compressed + // Returns: , whether file was compressed // Throws: , if compression is incorrect - compress(id, compression, cb) { + async compress(id, compression) { const file = this[getFilepath](id); - utils.compress(file, this.minCompressSize, compression, cb); + return utils.compress(file, this.minCompressSize, compression); } [getFilepath](id) { - return utils.getFilepath(this.dir, common.idToPath(id)); + return utils.getFilepath(this.path, common.idToPath(id)); } } @@ -139,14 +108,10 @@ class FileStorage { // options - // dir - , data storage directory // minCompressSize - , minimal file size to be compressed -// cb - , callback -// err - -// storage - -const create = (options, cb) => { - common.mkdirp(path.resolve(options.dir), err => { - if (err) cb(err); - else cb(null, new FileStorage(options)); - }); +// Returns: , instance +const create = async options => { + await utils.mkdirRecursive(path.resolve(options.path)); + return new FileStorage(options); }; module.exports = { FileStorage, create }; diff --git a/lib/fs.js b/lib/fs.js new file mode 100644 index 0000000..7e408c7 --- /dev/null +++ b/lib/fs.js @@ -0,0 +1,30 @@ +'use strict'; + +// TODO: this file should be removed and `fs.promises` used instead +// when support for Node.js 8 is dropped + +const fs = require('fs'); +const { promisify } = require('util'); + +const { iter } = require('@metarhia/common'); + +const list = [ + 'readFile', + 'writeFile', + 'unlink', + 'rename', + 'stat', + 'access', + 'mkdir', + 'rmdir', + 'readdir', +]; + +if (process.version.slice(1).split('.')[0] >= 10) { + module.exports = fs.promises; +} else { + module.exports = iter(list).collectWith( + {}, + (obj, name) => (obj[name] = promisify(fs[name])) + ); +} diff --git a/lib/utils.js b/lib/utils.js index 15eac1c..849641e 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,11 +1,12 @@ 'use strict'; -const fs = require('fs'); +const fs = require('./fs'); const path = require('path'); const crypto = require('crypto'); const { crc32 } = require('crc'); +const util = require('util'); +const common = require('@metarhia/common'); const { zip, gzip } = require('compressing'); -const metasync = require('metasync'); const CHECKSUM = 'CRC32'; const DEDUP_HASH = 'SHA256'; @@ -28,32 +29,6 @@ const FS_EXT = 'f'; const getFilepath = (...parts) => `${path.join(...parts)}.${FS_EXT}`; -const rmdirp = (dir, cb) => { - fs.stat(dir, (err, stats) => { - if (err) { - if (err.code === 'ENOENT') cb(null); - else cb(err); - return; - } - - if (stats.isDirectory()) { - fs.readdir(dir, (err, files) => { - if (err) { - cb(err); - return; - } - files = files.map(file => path.join(dir, file)); - metasync.each(files, rmdirp, err => { - if (err) cb(err); - else fs.rmdir(dir, cb); - }); - }); - } else { - fs.unlink(dir, cb); - } - }); -}; - const computeHash = (data, checksum) => { const hasher = hashers[checksum]; if (!hasher) { @@ -68,45 +43,57 @@ const getDataStats = (data, checksum, dedupHash) => ({ size: Buffer.byteLength(data), }); -const compress = (file, minCompressSize, compression, cb) => { - fs.stat(file, (err, stats) => { - if (err || stats.size <= minCompressSize) { - cb(err, false); - return; - } - const filec = file + 'z'; - const compressor = compressors[compression]; - if (!compressor) { - throw new Error(`Unknown compression type ${compression} specified`); - } - compressor - .compressFile(file, filec) - .then(() => - fs.rename(filec, file, err => { - if (err) cb(err, false); - else cb(null, true); - }) - ) - .catch(err => cb(err, false)); - }); +const mkdirRecursive = util.promisify(common.mkdirp); + +const rmRecursive = async dir => { + try { + await fs.access(dir); + } catch (err) { + if (err.code === 'ENOENT') return; + throw new Error(`Cannot access directory '${dir}': ${err.message}`); + } + + const files = await fs.readdir(dir); + + for (let file of files) { + file = path.join(dir, file); + const stats = await fs.stat(file); + + if (stats.isDirectory()) await rmRecursive(file); + else await fs.unlink(file); + } + + await fs.rmdir(dir); }; -const uncompress = (file, opts, cb) => { - fs.access(file, err => { - if (err) { - cb(err); - return; - } - const compressor = compressors[opts.compression]; - if (!compressor) { - throw new Error(`Unknown compression type ${opts.compression} specified`); - } +const compress = async (file, minCompressSize, compression) => { + const compressor = compressors[compression]; + if (!compressor) { + throw new Error(`Unknown compression type ${compression} specified`); + } + + const stats = await fs.stat(file); + if (stats.size <= minCompressSize) return false; + + const filec = file + 'z'; + await compressor.compressFile(file, filec); + await fs.rename(filec, file); + return true; +}; + +const uncompress = async (file, opts) => { + const compressor = compressors[opts.compression]; + if (!compressor) { + throw new Error(`Unknown compression type ${opts.compression} specified`); + } + + return new Promise((res, rej) => { const buffers = []; new compressor.UncompressStream({ source: file }) - .on('error', cb) + .on('error', rej) .on('finish', () => { - if (opts.encoding) cb(null, buffers.join()); - else cb(null, Buffer.concat(buffers)); + if (opts.encoding) res(buffers.join('')); + else res(Buffer.concat(buffers)); }) .on('entry', (header, stream, next) => { if (opts.encoding) stream.setEncoding(opts.encoding); @@ -120,7 +107,8 @@ module.exports = { getFilepath, computeHash, getDataStats, - rmdirp, + mkdirRecursive, + rmRecursive, compress, uncompress, }; diff --git a/package-lock.json b/package-lock.json index b93a561..4dbc6ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -216,22 +216,44 @@ "dev": true }, "cliui": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", - "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", "dev": true, "requires": { - "string-width": "^2.1.1", - "strip-ansi": "^4.0.0", - "wrap-ansi": "^2.0.0" + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } } }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -456,9 +478,9 @@ "dev": true }, "eslint-config-prettier": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-4.2.0.tgz", - "integrity": "sha512-y0uWc/FRfrHhpPZCYflWC8aE0KRJRY04rdZVfl8cL3sEZmOYyaBdhdlQPjKZBnuRMyLVK+JUZr7HaZFClQiH4w==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-4.3.0.tgz", + "integrity": "sha512-sZwhSTHVVz78+kYD3t5pCWSYEdVSBR0PXnwjDRsUs8ytIrK8PLXw+6FKp8r3Z7rx4ZszdetWlXYKOHoUrrwPlA==", "dev": true, "requires": { "get-stdin": "^6.0.0" @@ -519,9 +541,9 @@ } }, "eslint-plugin-import": { - "version": "2.17.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.17.2.tgz", - "integrity": "sha512-m+cSVxM7oLsIpmwNn2WXTJoReOF9f/CtLMo7qOVmKd1KntBy0hEcuNZ3erTmWjx+DxRO0Zcrm5KwAvI9wHcV5g==", + "version": "2.17.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.17.3.tgz", + "integrity": "sha512-qeVf/UwXFJbeyLbxuY8RgqDyEKCkqV7YC+E5S5uOjAp4tOc8zj01JP3ucoBM8JcEqd1qRasJSg6LLlisirfy0Q==", "dev": true, "requires": { "array-includes": "^3.0.3", @@ -534,7 +556,7 @@ "lodash": "^4.17.11", "minimatch": "^3.0.4", "read-pkg-up": "^2.0.0", - "resolve": "^1.10.0" + "resolve": "^1.11.0" }, "dependencies": { "debug": { @@ -565,9 +587,9 @@ } }, "eslint-plugin-prettier": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.0.1.tgz", - "integrity": "sha512-/PMttrarPAY78PLvV3xfWibMOdMDl57hmlQ2XqFeA37wd+CJ7WSxV7txqjVPHi/AAFKd2lX0ZqfsOc/i5yFCSQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.0.tgz", + "integrity": "sha512-XWX2yVuwVNLOUhQijAkXz+rMPPoCr7WFiAl8ig6I7Xn+pPVhDhzg4DxHpmbeb0iqjO9UronEA3Tb09ChnFVHHA==", "dev": true, "requires": { "prettier-linter-helpers": "^1.0.0" @@ -1138,14 +1160,14 @@ } }, "metatests": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/metatests/-/metatests-0.6.5.tgz", - "integrity": "sha512-+MEsUN9AOgYq61TMLcZrKOAPZflTbQcLuHZT55xFnBnpB2Y44yWLpJ+1YIznJh+Mz8Wqxv4VotHC7ONkORqQeg==", + "version": "0.6.6", + "resolved": "https://registry.npmjs.org/metatests/-/metatests-0.6.6.tgz", + "integrity": "sha512-rpdtT5muSymdLGYsyXjZHbm7wwjsbsn0hMbY50VOXxm+c8CVfw5uEY2VbC4u8Hxgvay3LP4MMZmATelBTIO9Kg==", "dev": true, "requires": { "@metarhia/common": "^1.4.2", "tap-mocha-reporter": "^4.0.1", - "yaml": "^1.5.0", + "tap-yaml": "^1.0.0", "yargs": "^13.2.2" }, "dependencies": { @@ -1240,12 +1262,6 @@ "path-key": "^2.0.0" } }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true - }, "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -1426,9 +1442,9 @@ "dev": true }, "prettier": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.17.0.tgz", - "integrity": "sha512-sXe5lSt2WQlCbydGETgfm1YBShgOX4HxQkFPvbxkcwgDvGDeqVau8h+12+lmSVlP3rHPz0oavfddSZg/q+Szjw==", + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz", + "integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==", "dev": true }, "prettier-linter-helpers": { @@ -1526,9 +1542,9 @@ "dev": true }, "resolve": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.1.tgz", - "integrity": "sha512-KuIe4mf++td/eFb6wkaPbMDnP6kObCaEtIDuHOUED6MNUo4K670KZUHuuvYPZDxNF0WVLw49n06M2m2dXphEzA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", + "integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==", "dev": true, "requires": { "path-parse": "^1.0.6" @@ -1957,48 +1973,40 @@ "dev": true }, "wrap-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", - "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", "dev": true, "requires": { - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1" + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" }, "dependencies": { "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" } }, "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "dev": true, "requires": { - "ansi-regex": "^2.0.0" + "ansi-regex": "^4.1.0" } } } @@ -2044,12 +2052,12 @@ } }, "yargs": { - "version": "13.2.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.2.tgz", - "integrity": "sha512-WyEoxgyTD3w5XRpAQNYUB9ycVH/PQrToaTXdYXRdOXvEy1l19br+VJsc0vcO8PTGg5ro/l/GY7F/JMEBmI0BxA==", + "version": "13.2.4", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.4.tgz", + "integrity": "sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg==", "dev": true, "requires": { - "cliui": "^4.0.0", + "cliui": "^5.0.0", "find-up": "^3.0.0", "get-caller-file": "^2.0.1", "os-locale": "^3.1.0", @@ -2059,7 +2067,7 @@ "string-width": "^3.0.0", "which-module": "^2.0.0", "y18n": "^4.0.0", - "yargs-parser": "^13.0.0" + "yargs-parser": "^13.1.0" }, "dependencies": { "ansi-regex": { diff --git a/package.json b/package.json index 4cb4914..dbe7d23 100644 --- a/package.json +++ b/package.json @@ -36,11 +36,11 @@ "devDependencies": { "eslint": "^5.16.0", "eslint-config-metarhia": "^7.0.0", - "eslint-config-prettier": "^4.1.0", - "eslint-plugin-import": "^2.17.2", - "eslint-plugin-prettier": "^3.0.1", + "eslint-config-prettier": "^4.3.0", + "eslint-plugin-import": "^2.17.3", + "eslint-plugin-prettier": "^3.1.0", "metasync": "^0.3.31", - "metatests": "^0.6.5", - "prettier": "^1.17.0" + "metatests": "^0.6.6", + "prettier": "^1.18.2" } } diff --git a/test/filestorage.js b/test/filestorage.js index 6e9e3ac..27be21b 100644 --- a/test/filestorage.js +++ b/test/filestorage.js @@ -1,9 +1,11 @@ 'use strict'; -const fs = require('fs'); +const fs = require('../lib/fs'); const path = require('path'); + const { create } = require('..'); const utils = require('../lib/utils'); + const metatests = require('metatests'); const common = require('@metarhia/common'); @@ -13,423 +15,244 @@ const minCompressSize = 1000; const fsTest = metatests.test('Filestorage'); fsTest.endAfterSubtests(); -fsTest.beforeEach((test, cb) => { - create({ dir: testDir, minCompressSize }, (err, storage) => { - test.error(err); - cb({ storage }); - }); -}); +let idCounter = 0; -fsTest.afterEach((test, cb) => { - utils.rmdirp(testDir, err => { - test.error(err); - cb(); - }); +fsTest.beforeEach(async () => { + const storage = await create({ path: testDir, minCompressSize }); + const id = new common.Uint64(idCounter++); + return { storage, id }; }); -fsTest.test('Create root directory on create', test => { - const dir = testDir + 'create'; - create({ dir }, err => { - test.error(err); - test.strictSame(fs.existsSync(dir), true); - utils.rmdirp(dir, err => { - test.error(err); - test.end(); - }); - }); -}); +fsTest.afterEach(() => utils.rmRecursive(testDir)); + +fsTest.test('Create root directory on create', () => fs.access(testDir)); -fsTest.test('Write string to file', (test, { storage }) => { +fsTest.test('Write string to file', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; const data = 'data0'; - const cs = '20cd1c30'; - const dh = '4285d10832a8edf4cc7979e9a257e145dc5c934549f62e0bf1dc6070e67cc4ab'; - const dataSize = 5; - - storage.write( - new common.Uint64(0), - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - fs.readFile(utils.getFilepath(testDir, '0000', '0000'), (err, d) => { - test.error(err); - test.strictSame(d.toString(), data); - test.end(); - }); - } - ); + const dataStats = { + checksum: '20cd1c30', + dedupHash: + '4285d10832a8edf4cc7979e9a257e145dc5c934549f62e0bf1dc6070e67cc4ab', + size: 5, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(fs.readFile(file, 'utf8'), data); }); -fsTest.test('Write Buffer to file', (test, { storage }) => { - const data = 'data1'; - const cs = '57ca2ca6'; - const dh = '5b41362bc82b7f3d56edc5a306db22105707d01ff4819e26faef9724a2d406c9'; - const dataSize = 5; - - storage.write( - new common.Uint64(1), - Buffer.from(data), - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - fs.readFile(utils.getFilepath(testDir, '0001', '0000'), (err, d) => { - test.error(err); - test.strictSame(d.toString(), data); - test.end(); - }); - } - ); +fsTest.test('Write Buffer to file', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = Buffer.from('data1'); + const dataStats = { + checksum: '57ca2ca6', + dedupHash: + '5b41362bc82b7f3d56edc5a306db22105707d01ff4819e26faef9724a2d406c9', + size: 5, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(fs.readFile(file), data); }); -fsTest.test('Read file', (test, { storage }) => { +fsTest.test('Read file', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data2'; - fs.mkdir(path.join(testDir, '0002'), err => { - test.error(err); - - fs.writeFile(utils.getFilepath(testDir, '0002', '0000'), data, err => { - test.error(err); - - storage.read(new common.Uint64(2), {}, (err, d) => { - test.error(err); - test.strictSame(d.toString(), data); - test.end(); - }); - }); - }); + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await test.resolves(storage.read(id, { encoding: 'utf8' }), data); }); -fsTest.test('Remove file', (test, { storage }) => { +fsTest.test('Remove file', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data3'; - const file = utils.getFilepath(testDir, '0003', '0000'); - - fs.mkdir(path.join(testDir, '0003'), err => { - test.error(err); - fs.writeFile(file, data, err => { - test.error(err); + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await storage.rm(id); - storage.rm(new common.Uint64(3), err => { - test.error(err); - test.strictSame(fs.existsSync(file), false); - test.end(); - }); - }); - }); + const err = await test.rejects(fs.access(file)); + test.strictSame(err.code, 'ENOENT'); }); -fsTest.test('Compress small file using zip', (test, { storage }) => { +fsTest.test('Compress small file using zip', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data4'; - const file = utils.getFilepath(testDir, '0004', '0000'); - - fs.mkdir(path.join(testDir, '0004'), err => { - test.error(err); - - fs.writeFile(utils.getFilepath(testDir, '0004', '0000'), data, err => { - test.error(err); - - storage.compress(new common.Uint64(4), 'ZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, false); - - fs.readFile(file, 'utf8', (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - }); - }); - }); + + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await test.resolves(storage.compress(id, 'ZIP'), false); + await test.resolves(fs.readFile(file, 'utf8'), data); }); -fsTest.test('Compress small file using gzip', (test, { storage }) => { +fsTest.test('Compress small file using gzip', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data5'; - const file = utils.getFilepath(testDir, '0005', '0000'); - - fs.mkdir(path.join(testDir, '0005'), err => { - test.error(err); - - fs.writeFile(file, data, err => { - test.error(err); - - storage.compress(new common.Uint64(5), 'GZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, false); - - fs.readFile(file, 'utf8', (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - }); - }); - }); + + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await test.resolves(storage.compress(id, 'GZIP'), false); + await test.resolves(fs.readFile(file, 'utf8'), data); }); -fsTest.test('Compress big file using zip', (test, { storage }) => { +fsTest.test('Compress big file using zip', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data6'.repeat(1000); - const file = utils.getFilepath(testDir, '0006', '0000'); - - fs.mkdir(path.join(testDir, '0006'), err => { - test.error(err); - - fs.writeFile(file, data, err => { - test.error(err); - - storage.compress(new common.Uint64(6), 'ZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, true); - test.strictSame(fs.statSync(file).size < 5000, true); - - fs.readFile(file, 'utf8', (err, d) => { - test.error(err); - test.strictNotSame(d, data); - test.end(); - }); - }); - }); - }); + + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await test.resolves(storage.compress(id, 'ZIP'), true); + + const d = await fs.readFile(file, 'utf8'); + test.strictNotSame(d, data); + + const { size } = await fs.stat(file); + test.assert(size < 5000); }); -fsTest.test('Compress big file using gzip', (test, { storage }) => { +fsTest.test('Compress big file using gzip', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data7'.repeat(1000); - const file = utils.getFilepath(testDir, '0007', '0000'); - - fs.mkdir(path.join(testDir, '0007'), err => { - test.error(err); - - fs.writeFile(file, data, err => { - test.error(err); - - storage.compress(new common.Uint64(7), 'GZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, true); - test.strictSame(fs.statSync(file).size < 5000, true); - - fs.readFile(file, 'utf8', (err, d) => { - test.error(err); - test.strictNotSame(d, data); - test.end(); - }); - }); - }); - }); -}); -fsTest.test('Write and read small file', (test, { storage }) => { - const id = new common.Uint64(8); - const data = 'data8'; - const cs = '2e169402'; - const dh = 'b5cc74ab5bb5a5f1acc7407be3e4cbce8611c5ed07354ab9e510b74ee0b273cb'; - const dataSize = 5; - - storage.write( - id, - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - storage.read(id, { encoding: 'utf8' }, (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - } - ); -}); + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); + await test.resolves(storage.compress(id, 'GZIP'), true); -fsTest.test('Write, compress and read small file', (test, { storage }) => { - const id = new common.Uint64(9); - const data = 'data9'; - const cs = '5911a494'; - const dh = 'bbe0aa41024faeac81813a0194a95637d54cc65c025e0efd857ce0afcd51573f'; - const dataSize = 5; - - storage.write( - id, - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - storage.compress(id, 'ZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, false); - - storage.read(id, { encoding: 'utf8' }, (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - }); - } - ); -}); + const d = await fs.readFile(file, 'utf8'); + test.strictNotSame(d, data); -fsTest.test('Write and read big file', (test, { storage }) => { - const id = new common.Uint64(10); - const data = 'data10'.repeat(1000); - const cs = '828aaf45'; - const dh = '7a8553c5934fd3b4a068a3d4c5bafc189e1fe8e0f7f46cbcc0fc1199249a39a2'; - const dataSize = 6000; - - storage.write( - id, - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - storage.read(id, { encoding: 'utf8' }, (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - } - ); + const { size } = await fs.stat(file); + test.assert(size < 5000); }); -fsTest.test('Write, compress and read big file', (test, { storage }) => { - const id = new common.Uint64(11); - const data = 'data11'.repeat(1000); - const cs = '5928f9a'; - const dh = 'f1cf91dea01f77fd860645de51462794cfbbae0a14d81350b21934e58a69941d'; - const dataSize = 6000; - - storage.write( - id, - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - storage.compress(id, 'ZIP', (err, compressed) => { - test.error(err); - test.strictSame(compressed, true); - - storage.read(id, { encoding: 'utf8', compression: 'ZIP' }, (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - }); - } - ); +fsTest.test('Write and read small file', async (test, { storage, id }) => { + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = 'data8'; + const dataStats = { + checksum: '2e169402', + dedupHash: + 'b5cc74ab5bb5a5f1acc7407be3e4cbce8611c5ed07354ab9e510b74ee0b273cb', + size: 5, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(storage.read(id, { encoding: 'utf8' }), data); }); -fsTest.test('Write and read small file', (test, { storage }) => { - const id = new common.Uint64(12); - const data = '白い猫でも黒い猫でも鼠を取る猫はいい猫だ'; - const cs = '289722d'; - const dh = 'fab270bfcdc81e464611cd112e95c6b9590ba55483fbfe2dda98d67ddc741ab3'; - const dataSize = 60; - - storage.write( - id, - data, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs); - test.strictSame(dedupHash, dh); - test.strictSame(size, dataSize); - - storage.read(id, { encoding: 'utf8' }, (err, d) => { - test.error(err); - test.strictSame(d, data); - test.end(); - }); - } - ); +fsTest.test( + 'Write, compress, read small file', + async (test, { storage, id }) => { + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = 'data9'; + const dataStats = { + checksum: '5911a494', + dedupHash: + 'bbe0aa41024faeac81813a0194a95637d54cc65c025e0efd857ce0afcd51573f', + size: 5, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(storage.compress(id, 'ZIP'), false); + await test.resolves(storage.read(id, { encoding: 'utf8' }), data); + } +); + +fsTest.test('Write and read big file', async (test, { storage, id }) => { + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = 'data10'.repeat(1000); + const dataStats = { + checksum: '828aaf45', + dedupHash: + '7a8553c5934fd3b4a068a3d4c5bafc189e1fe8e0f7f46cbcc0fc1199249a39a2', + size: 6000, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(storage.read(id, { encoding: 'utf8' }), data); }); -fsTest.test('Get file stats', (test, { storage }) => { +fsTest.test( + 'Write, compress and read big file', + async (test, { storage, id }) => { + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = 'data11'.repeat(1000); + const dataStats = { + checksum: '5928f9a', + dedupHash: + 'f1cf91dea01f77fd860645de51462794cfbbae0a14d81350b21934e58a69941d', + size: 6000, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(storage.compress(id, 'ZIP'), true); + await test.resolves( + storage.read(id, { encoding: 'utf8', compression: 'ZIP' }), + data + ); + } +); + +fsTest.test( + 'Write and read small unicode file', + async (test, { storage, id }) => { + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; + const data = '白い猫でも黒い猫でも鼠を取る猫はいい猫だ'; + const dataStats = { + checksum: '289722d', + dedupHash: + 'fab270bfcdc81e464611cd112e95c6b9590ba55483fbfe2dda98d67ddc741ab3', + size: 60, + }; + + await test.resolves(storage.write(id, data, opts), dataStats); + await test.resolves(storage.read(id, { encoding: 'utf8' }), data); + } +); + +fsTest.test('Get file stats', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); const data = 'data13'; - const file = utils.getFilepath(testDir, '000d', '0000'); - - fs.mkdir(path.join(testDir, '000d'), err => { - test.error(err); - fs.writeFile(file, data, err => { - test.error(err); + await fs.mkdir(path.join(testDir, pathPart)); + await fs.writeFile(file, data); - fs.stat(file, (err, fstats) => { - test.error(err); - - storage.stat(new common.Uint64(13), (err, stats) => { - test.error(err); - test.strictSame(stats, fstats); - test.end(); - }); - }); - }); - }); + const fstats = await fs.stat(file); + await test.resolves(storage.stat(id), fstats); }); -fsTest.test('Update file', (test, { storage }) => { +fsTest.test('Update file', async (test, { storage, id }) => { + const pathPart = id.toString(16).padStart(4, '0'); + const file = utils.getFilepath(testDir, pathPart, '0000'); + const opts = { checksum: 'CRC32', dedupHash: 'SHA256' }; const data1 = 'data15'; - const cs1 = 'bb53e75f'; - const dh1 = - '87fef323635f22fc88999957a585c3cf3f8383029c8501e52928a14028c0af2c'; - const dataSize1 = 6; - const data2 = 'another data'; - const cs2 = '963fe1ef'; - const dh2 = - 'f3cfa0c4064755101ffbcdc8a8d1b9dccc46d45b3a82f800a6eaab42e65f14c9'; - const dataSize2 = 12; - - const id = new common.Uint64(15); - - storage.write( - id, - data1, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size }) => { - test.error(err); - test.strictSame(checksum, cs1); - test.strictSame(dedupHash, dh1); - test.strictSame(size, dataSize1); - - storage.update( - id, - data2, - { checksum: 'CRC32', dedupHash: 'SHA256' }, - (err, { checksum, dedupHash, size, originalSize }) => { - test.error(err); - test.strictSame(checksum, cs2); - test.strictSame(dedupHash, dh2); - test.strictSame(size, dataSize2); - test.strictSame(originalSize, dataSize1); - - fs.readFile(utils.getFilepath(testDir, '000f', '0000'), (err, d) => { - test.error(err); - test.strictSame(d.toString(), data2); - test.end(); - }); - } - ); - } - ); + const dataStats1 = { + checksum: 'bb53e75f', + dedupHash: + '87fef323635f22fc88999957a585c3cf3f8383029c8501e52928a14028c0af2c', + size: 6, + }; + const dataStats2 = { + checksum: '963fe1ef', + dedupHash: + 'f3cfa0c4064755101ffbcdc8a8d1b9dccc46d45b3a82f800a6eaab42e65f14c9', + size: 12, + originalSize: 6, + }; + + await test.resolves(storage.write(id, data1, opts), dataStats1); + await test.resolves(storage.update(id, data2, opts), dataStats2); + await test.resolves(fs.readFile(file, 'utf8'), data2); }); diff --git a/test/utils.js b/test/utils.js index 9bd6f44..03ca6a9 100644 --- a/test/utils.js +++ b/test/utils.js @@ -1,10 +1,9 @@ 'use strict'; -const fs = require('fs'); +const fs = require('../lib/fs'); const path = require('path'); const utils = require('../lib/utils'); const metatests = require('metatests'); -const common = require('@metarhia/common'); const testDir = path.join(__dirname, 'utils-test-root'); @@ -69,37 +68,21 @@ metatests.testSync('getDataStats', test => { test.strictSame(stats.size, dataSize); }); -const finish = test => { - utils.rmdirp(testDir, err => { - test.error(err); - test.end(); - }); -}; - -metatests.test('', test => { +metatests.test('Compress and uncompress file', async test => { const dir = path.join(testDir, 'some', 'path', 'to', 'nested', 'dir'); - common.mkdirp(dir, err => { - test.error(err); - test.strictSame(fs.existsSync(dir), true); + await utils.mkdirRecursive(dir); + await fs.access(testDir); + + const file = path.join(testDir, 'file'); + const data = 'data'.repeat(1000); + const opts = { compression: 'ZIP', encoding: 'utf8' }; + + await fs.writeFile(file, data); + await utils.compress(file, 1024, 'ZIP'); - const file = path.join(testDir, 'file'); - const data = 'data'.repeat(1000); - fs.writeFile(file, data, err => { - test.error(err); - utils.compress(file, 1024, 'ZIP', err => { - test.error(err); - test.strictSame(fs.statSync(file).size < 4000, true); + const { size } = await fs.stat(file); + test.assert(size < data.length); - utils.uncompress( - file, - { compression: 'ZIP', encoding: 'utf8' }, - (err, d) => { - test.error(err); - test.strictSame(d, data); - finish(test); - } - ); - }); - }); - }); + await test.resolves(utils.uncompress(file, opts), data); + await utils.rmRecursive(testDir); });