diff --git a/package-lock.json b/package-lock.json index 1c8c1a9..c870550 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "r2-utils-js", + "name": "@evidentpoint/r2-utils-js", "version": "1.0.0-alpha.1", "lockfileVersion": 1, "requires": true, @@ -47,6 +47,12 @@ "integrity": "sha512-orGL5LXERPYsLov6CWs3Fh6203+dXzJkR7OnddIr2514Hsecwc8xRpzCapshBbKFImCsvS/mk6+FWiN5LyZJAQ==", "dev": true }, + "@types/events": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", + "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", + "dev": true + }, "@types/form-data": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", @@ -56,10 +62,27 @@ "@types/node": "*" } }, + "@types/glob": { + "version": "5.0.35", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-5.0.35.tgz", + "integrity": "sha512-wc+VveszMLyMWFvXLkloixT4n0harUIVZjnpzztaZ0nKLuul7Z32iMt2fUFGAaZ4y1XWjFRMtCI5ewvyh4aIeg==", + "dev": true, + "requires": { + "@types/events": "*", + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, "@types/node": { - "version": "8.10.20", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.20.tgz", - "integrity": "sha512-M7x8+5D1k/CuA6jhiwuSCmE8sbUWJF0wYsjcig9WrXvwUI5ArEoUBdOXpV4JcEMrLp02/QbDjw+kI+vQeKyQgg==", + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.3.4.tgz", + "integrity": "sha512-YMLlzdeNnAyLrQew39IFRkMacAR5BqKGIEei9ZjdHsIZtv+ZWKYTu1i7QJhetxQ9ReXx8w5f+cixdHZG3zgMQA==", "dev": true }, "@types/request": { @@ -400,6 +423,11 @@ } } }, + "base64-js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + }, "bcrypt-pbkdf": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", @@ -472,6 +500,15 @@ } } }, + "buffer": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.1.0.tgz", + "integrity": "sha512-YkIRgwsZwJWTnyQrsBTWefizHh+8GYj3kbL1BTiAQ/9pwpino0G7B2gp5tx/FUBqUlvtxV85KNR3mwfAtv15Yw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -489,6 +526,21 @@ "integrity": "sha1-A1O1T9B/2VZBcGca5vZrnPENJ/U=", "dev": true }, + "buffer-image-size": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/buffer-image-size/-/buffer-image-size-0.6.4.tgz", + "integrity": "sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==", + "requires": { + "@types/node": "*" + }, + "dependencies": { + "@types/node": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.3.4.tgz", + "integrity": "sha512-YMLlzdeNnAyLrQew39IFRkMacAR5BqKGIEei9ZjdHsIZtv+ZWKYTu1i7QJhetxQ9ReXx8w5f+cixdHZG3zgMQA==" + } + } + }, "buffer-indexof-polyfill": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz", @@ -901,9 +953,9 @@ } }, "cpy": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/cpy/-/cpy-7.0.0.tgz", - "integrity": "sha512-TrCxDbSjKZyz/jgmelcIh3meeSiiwzb561m3pYEDnzkAN4J6OijAt8cL1j3vvHG4ER/HgM4YMHKb5kyR5y5Hqw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cpy/-/cpy-7.0.1.tgz", + "integrity": "sha512-Zo52tXKLJcgy/baacn6KaNoRAakkl2wb+R4u6qJ4wlD0uchncwRQcIk66PlGlkzuToCJO6A6PWX27Tdwc8LU2g==", "dev": true, "requires": { "arrify": "^1.0.1", @@ -932,6 +984,15 @@ "is-windows": "^1.0.0" } }, + "cross-fetch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-2.2.1.tgz", + "integrity": "sha1-lshZEE113vyWf7XbYkdOdUJrArA=", + "requires": { + "node-fetch": "2.1.2", + "whatwg-fetch": "2.0.4" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -1481,6 +1542,19 @@ "pend": "~1.2.0" } }, + "fetch-ponyfill": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/fetch-ponyfill/-/fetch-ponyfill-6.0.2.tgz", + "integrity": "sha512-hBWVb6Z0chjr9r0g4N5uknldAnyFstWTnX6oFxMsL9ZlSL7fa66Keis3y4SCMgpTggY4GeGbB4z3z5g5SmrgkQ==", + "requires": { + "node-fetch": "~2.1.0" + } + }, + "fetch-readablestream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.1.0.tgz", + "integrity": "sha1-Ihm8zqm/O5fLWZYkCUCZ6vqkAL4=" + }, "file-js": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/file-js/-/file-js-0.3.0.tgz", @@ -2038,10 +2112,15 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz", + "integrity": "sha512-GguP+DRY+pJ3soyIiGPTvdiVXjZ+DbXOxGpXn3eMvNW4x4irjqXm4wHKscC+TfxSJ0yw/S1F24tqdMNsMZTiLA==" + }, "ignore": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.8.tgz", - "integrity": "sha512-pUh+xUQQhQzevjRHHFqqcTy0/dP/kS9I8HSrUydhihjuD09W6ldVWFtIrwhXdUJHis3i2rZNqEHpZH/cbinFbg==", + "version": "3.3.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", "dev": true }, "in-gfw": { @@ -2296,8 +2375,7 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", - "dev": true + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, "is-typedarray": { "version": "1.0.0", @@ -2354,6 +2432,23 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, + "isomorphic-fetch-readablestream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch-readablestream/-/isomorphic-fetch-readablestream-1.1.0.tgz", + "integrity": "sha512-onqnFkasii47ze4OUldKDNtpvZm9kh8Q4T1f1H3kzBTWzUmmmqo9T9B37b7Tq46V4O0zwLxJbqj/y3BGFOGWQQ==", + "requires": { + "fetch-readablestream": "^0.1.0", + "node-fetch": "^2.1.1" + } + }, + "isomorphic-url-shim": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/isomorphic-url-shim/-/isomorphic-url-shim-1.0.2.tgz", + "integrity": "sha512-C4n3d3YJ7mi8toCn3Ky4HmHyOhnNPiy/ok/ozX+SrI7kGlnty6Qs5HrgDwrPhGfhe2iouyypo3met8YDSX/8Yw==", + "requires": { + "url-ponyfill": "^0.5.9" + } + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -2628,9 +2723,9 @@ } }, "mem": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mem/-/mem-3.0.0.tgz", - "integrity": "sha1-hOWK1N+99dEFsmtlSKOYsrOqiiE=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mem/-/mem-3.0.1.tgz", + "integrity": "sha512-QKs47bslvOE0NbXOqG6lMxn6Bk0Iuw0vfrIeLykmQle2LkCw1p48dZDdzE+D88b/xqRJcZGcMNeDvSVma+NuIQ==", "dev": true, "requires": { "mimic-fn": "^1.0.0", @@ -2821,6 +2916,11 @@ "integrity": "sha512-2NpiFHqC87y/zFke0fC0spBXL3bBsoh/p5H1EFhshxjCR5+0g2d6BiXbUFz9v1sAcxsk2htp2eQnNIci2dIYcA==", "dev": true }, + "node-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" + }, "node-stream-zip": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.7.0.tgz", @@ -3249,6 +3349,11 @@ "util-deprecate": "~1.0.1" } }, + "readable-stream-node-to-web": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/readable-stream-node-to-web/-/readable-stream-node-to-web-1.0.1.tgz", + "integrity": "sha1-i3YU+qFGXr+g2pucpjA/onBzt88=" + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -3886,8 +3991,8 @@ } }, "ta-json": { - "version": "github:danielweck/ta-json#9938af9c192ca36aa600f2ce14eb5735aa1a50e6", - "from": "github:danielweck/ta-json#dist", + "version": "github:evidentpoint/ta-json#0b23daae9b78607561110196859d9cfdd1f23300", + "from": "github:evidentpoint/ta-json#dist", "requires": { "reflect-metadata": "^0.1.12" }, @@ -3908,6 +4013,11 @@ "execa": "^0.7.0" } }, + "text-encoding": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", + "integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk=" + }, "through2": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz", @@ -4030,9 +4140,9 @@ "dev": true }, "tslib": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.2.tgz", - "integrity": "sha512-AVP5Xol3WivEr7hnssHDsaM+lVrVXWUvd1cfXTRkTj80b//6g2wIFEH6hZG0muGZRnHGrfttpdzRk3YlBkWjKw==" + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", + "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==" }, "tslint": { "version": "5.10.0", @@ -4161,9 +4271,9 @@ } }, "universalify": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", - "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", "dev": true }, "unset-value": { @@ -4244,6 +4354,11 @@ "prepend-http": "^2.0.0" } }, + "url-ponyfill": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/url-ponyfill/-/url-ponyfill-0.5.9.tgz", + "integrity": "sha1-ZNKKs1p8kfBCpGIhU1vip/f3Gzc=" + }, "url-to-options": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", @@ -4296,9 +4411,9 @@ } }, "vinyl": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.1.0.tgz", - "integrity": "sha1-Ah+cLPlR1rk5lDyJ617lrdT9kkw=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz", + "integrity": "sha512-MBH+yP0kC/GQ5GwBqrTPTzEfiiLjta7hTtvQtbxBgTeSXsmKQRQecjibMbxIXzVT3Y9KJK+drOz1/k+vsu8Nkg==", "dev": true, "requires": { "clone": "^2.1.1", @@ -4381,6 +4496,26 @@ "vinyl": "^2.0.0" } }, + "web-streams-node": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/web-streams-node/-/web-streams-node-0.3.1.tgz", + "integrity": "sha512-t9hOf4EyX0fqPbAYyAI5jC+1NzlL0gI7ZVplewcB2f4WVgHVUmYDZb0YyjlWXp6B61sxnBq/udvKwJAseehSnQ==", + "requires": { + "is-stream": "^1.1.0", + "readable-stream-node-to-web": "^1.0.1", + "web-streams-ponyfill": "^1.4.1" + } + }, + "web-streams-ponyfill": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/web-streams-ponyfill/-/web-streams-ponyfill-1.4.2.tgz", + "integrity": "sha512-LCHW+fE2UBJ2vjhqJujqmoxh1ytEDEr0dPO3CabMdMDJPKmsaxzS90V1Ar6LtNE5VHLqxR4YMEj1i4lzMAccIA==" + }, + "whatwg-fetch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz", + "integrity": "sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index 8fbbdba..bd897b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "r2-utils-js", - "version": "1.0.0-alpha.1", + "name": "@evidentpoint/r2-utils-js", + "version": "1.0.0-alpha.2", "description": "Readium 2 'utils' for NodeJS (TypeScript)", "keywords": [ "readium", @@ -46,22 +46,33 @@ }, "homepage": "https://github.com/edrlab/r2-utils-js", "dependencies": { + "buffer": "^5.1.0", + "buffer-image-size": "^0.6.4", + "cross-fetch": "^2.1.0", "debug": "^3.1.0", + "fetch-ponyfill": "^6.0.1", "filehound": "^1.16.4", + "glob": "^7.1.2", + "isomorphic-fetch-readablestream": "^1.1.0", + "isomorphic-url-shim": "^1.0.2", "node-stream-zip": "^1.7.0", "reflect-metadata": "^0.1.12", "request": "^2.87.0", "request-promise-native": "^1.0.5", - "ta-json": "github:danielweck/ta-json#dist", + "ta-json": "github:evidentpoint/ta-json#dist", + "text-encoding": "^0.6.4", "tslib": "^1.9.2", "unzipper": "^0.8.14", + "web-streams-node": "^0.3.1", + "web-streams-ponyfill": "^1.4.2", "xpath": "^0.0.27", "yauzl": "^2.9.2", "yazl": "^2.4.3" }, "devDependencies": { "@types/debug": "^0.0.30", - "@types/node": "8.*.*", + "@types/glob": "^5.0.35", + "@types/node": "^10.3.4", "@types/request": "^2.47.1", "@types/request-promise-native": "^1.0.15", "cpy-cli": "^2.0.0", diff --git a/src/_utils/http/HttpReadableStream.ts b/src/_utils/http/HttpReadableStream.ts index 8131be4..5635559 100644 --- a/src/_utils/http/HttpReadableStream.ts +++ b/src/_utils/http/HttpReadableStream.ts @@ -12,6 +12,7 @@ import * as request from "request"; import * as requestPromise from "request-promise-native"; import { streamToBufferPromise } from "@utils/stream/BufferUtils"; +import { toWebReadableStream } from "web-streams-node"; ///////////// ///////////// @@ -65,7 +66,7 @@ export class HttpReadableStream extends Readable { let buffer: Buffer; try { - buffer = await streamToBufferPromise(res); + buffer = await streamToBufferPromise(toWebReadableStream(res)); } catch (err) { failure(err); return; diff --git a/src/_utils/http/UrlUtils.ts b/src/_utils/http/UrlUtils.ts index c2ecfb1..73c7e64 100644 --- a/src/_utils/http/UrlUtils.ts +++ b/src/_utils/http/UrlUtils.ts @@ -10,7 +10,12 @@ import * as querystring from "querystring"; export function isHTTP(urlOrPath: string): boolean { // TODO: smarter regexp - return urlOrPath.indexOf("http") === 0; + return !!urlOrPath.match(/^http:\/\/.*/); +} + +export function isDirectory(urlOrPath: string): boolean { + // TODO: smarter regexp + return !!urlOrPath.match(new RegExp(`.*${path.sep}\$`)); } export function encodeURIComponent_RFC3986(str: string): string { diff --git a/src/_utils/image-size/index.ts b/src/_utils/image-size/index.ts new file mode 100644 index 0000000..0163567 --- /dev/null +++ b/src/_utils/image-size/index.ts @@ -0,0 +1,2 @@ +import * as imageSize from "buffer-image-size"; +export default imageSize; diff --git a/src/_utils/perf-cli.ts b/src/_utils/perf-cli.ts index 5ab89fc..6ada91e 100644 --- a/src/_utils/perf-cli.ts +++ b/src/_utils/perf-cli.ts @@ -8,13 +8,13 @@ import * as fs from "fs"; import * as path from "path"; -import { IStreamAndLength, IZip } from "@utils/zip/zip"; -import { ZipExploded } from "@utils/zip/zip-ex"; -import { Zip1 } from "@utils/zip/zip1"; -import { Zip2 } from "@utils/zip/zip2"; -import { Zip3 } from "@utils/zip/zip3"; +// import { IStreamAndLength, IZip } from "@utils/zip/zip"; +// import { ZipExploded } from "@utils/zip/zip-ex"; +// import { Zip1 } from "@utils/zip/zip1"; +// import { Zip2 } from "@utils/zip/zip2"; +// import { Zip3 } from "@utils/zip/zip3"; -import { streamToBufferPromise } from "@utils/stream/BufferUtils"; +// import { streamToBufferPromise } from "@utils/stream/BufferUtils"; console.log("process.cwd():"); console.log(process.cwd()); @@ -57,56 +57,56 @@ const ext = path.extname(fileName).toLowerCase(); if (stats.isDirectory()) { // tslint:disable-next-line:no-floating-promises - (async () => { - const zipExploded: IZip = await ZipExploded.loadPromise(filePath); - const entries = await zipExploded.getEntries(); - for (const entryName of entries) { - console.log("############## " + entryName); - - let zipStream_: IStreamAndLength; - try { - zipStream_ = await zipExploded.entryStreamPromise(entryName); - } catch (err) { - console.log(err); - return; - } - const zipStream = zipStream_.stream; - let zipData: Buffer; - try { - zipData = await streamToBufferPromise(zipStream); - } catch (err) { - console.log(err); - return; - } - - if (entryName.endsWith(".css")) { - const str = zipData.toString("utf8"); - console.log(str); - } - } - })(); + // (async () => { + // const zipExploded: IZip = await ZipExploded.loadPromise(filePath); + // const entries = await zipExploded.getEntries(); + // for (const entryName of entries) { + // console.log("############## " + entryName); + // + // let zipStream_: IStreamAndLength; + // try { + // zipStream_ = await zipExploded.entryStreamPromise(entryName); + // } catch (err) { + // console.log(err); + // return; + // } + // const zipStream = zipStream_.stream; + // let zipData: Buffer; + // try { + // zipData = await streamToBufferPromise(zipStream); + // } catch (err) { + // console.log(err); + // return; + // } + // + // if (entryName.endsWith(".css")) { + // const str = zipData.toString("utf8"); + // console.log(str); + // } + // } + // })(); } else if (/\.epub[3]?$/.test(ext) || ext === ".cbz" || ext === ".zip") { // tslint:disable-next-line:no-floating-promises - (async () => { - const time3 = process.hrtime(); - const zip3: IZip = await Zip3.loadPromise(filePath); - const diff3 = process.hrtime(time3); - console.log(`Zip 3 (${zip3.entriesCount()}): ${diff3[0]} seconds + ${diff3[1]} nanoseconds`); - - const time2 = process.hrtime(); - const zip2: IZip = await Zip2.loadPromise(filePath); - const diff2 = process.hrtime(time2); - console.log(`Zip 2 (${zip2.entriesCount()}): ${diff2[0]} seconds + ${diff2[1]} nanoseconds`); - - const time1 = process.hrtime(); - const zip1: IZip = await Zip1.loadPromise(filePath); - const diff1 = process.hrtime(time1); - // const nanos = diff1[0] * 1e9 + diff1[1]; - console.log(`Zip 1 (${zip1.entriesCount()}): ${diff1[0]} seconds + ${diff1[1]} nanoseconds`); - - // const entries = await zip1.getEntries(); - // for (const entryName of entries) { - // console.log(entryName); - // } - })(); + // (async () => { + // const time3 = process.hrtime(); + // const zip3: IZip = await Zip3.loadPromise(filePath); + // const diff3 = process.hrtime(time3); + // console.log(`Zip 3 (${zip3.entriesCount()}): ${diff3[0]} seconds + ${diff3[1]} nanoseconds`); + // + // const time2 = process.hrtime(); + // const zip2: IZip = await Zip2.loadPromise(filePath); + // const diff2 = process.hrtime(time2); + // console.log(`Zip 2 (${zip2.entriesCount()}): ${diff2[0]} seconds + ${diff2[1]} nanoseconds`); + // + // const time1 = process.hrtime(); + // const zip1: IZip = await Zip1.loadPromise(filePath); + // const diff1 = process.hrtime(time1); + // // const nanos = diff1[0] * 1e9 + diff1[1]; + // console.log(`Zip 1 (${zip1.entriesCount()}): ${diff1[0]} seconds + ${diff1[1]} nanoseconds`); + // + // // const entries = await zip1.getEntries(); + // // for (const entryName of entries) { + // // console.log(entryName); + // // } + // })(); } diff --git a/src/_utils/stream/BufferUtils.ts b/src/_utils/stream/BufferUtils.ts index 506f8c1..04e66d1 100644 --- a/src/_utils/stream/BufferUtils.ts +++ b/src/_utils/stream/BufferUtils.ts @@ -6,9 +6,17 @@ // ==LICENSE-END== import { BufferReadableStream } from "@utils/stream/BufferReadableStream"; +// import { toWebReadableStream } from "web-streams-node"; // import { PassThrough } from "stream"; -export function bufferToStream(buffer: Buffer): NodeJS.ReadableStream { +export function bufferToStream(buffer: Buffer): ReadableStream { + + // return toWebReadableStream(buffer); + console.log(buffer); + return new ReadableStream(); +} + +export function bufferToNodeStream(buffer: Buffer): NodeJS.ReadableStream { return new BufferReadableStream(buffer); @@ -75,32 +83,20 @@ export function bufferToStream(buffer: Buffer): NodeJS.ReadableStream { // return stream; } -export async function streamToBufferPromise_READABLE(readStream: NodeJS.ReadableStream): Promise { - - return new Promise((resolve, reject) => { - - const buffers: Buffer[] = []; - - readStream.on("error", reject); - - readStream.on("readable", () => { - let chunk: Buffer; - do { - chunk = readStream.read() as Buffer; - if (chunk) { - buffers.push(chunk); - } - } - while (chunk); - }); +async function pump(reader: ReadableStreamReader, buffers: Buffer[]): Promise { + const next = await reader.read(); + if (next.done) { + return Buffer.concat(buffers); + } + buffers.push(Buffer.from(next.value)); + return pump(reader, buffers); +} - readStream.on("end", () => { - resolve(Buffer.concat(buffers)); - }); - }); +export async function streamToBufferPromise(readableStream: ReadableStream): Promise { + return pump(readableStream.getReader(), []); } -export async function streamToBufferPromise(readStream: NodeJS.ReadableStream): Promise { +export async function nodeStreamToBufferPromise(readStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { diff --git a/src/_utils/ta-json/JsonPropertyEx.ts b/src/_utils/ta-json/JsonPropertyEx.ts index 107e062..65f02c8 100644 --- a/src/_utils/ta-json/JsonPropertyEx.ts +++ b/src/_utils/ta-json/JsonPropertyEx.ts @@ -9,9 +9,9 @@ import "reflect-metadata"; import * as util from "util"; +import { getDefinition } from "@ta-json/classes/object-definition"; import * as debug_ from "debug"; import { JsonProperty } from "ta-json"; -import { getDefinition } from "ta-json/classes/object-definition"; const debug = debug_("r2:utils#ta-json/JsonPropertyEx"); diff --git a/src/_utils/zip/zip-ex.ts b/src/_utils/zip/zip-ex.ts index 0651719..09789b5 100644 --- a/src/_utils/zip/zip-ex.ts +++ b/src/_utils/zip/zip-ex.ts @@ -10,6 +10,8 @@ import * as filehound from "filehound"; import * as fs from "fs"; import * as path from "path"; +import { toWebReadableStream } from "web-streams-node"; + import { IStreamAndLength, IZip, Zip } from "./zip"; // import { bufferToStream } from "../stream/BufferUtils"; @@ -38,7 +40,7 @@ export class ZipExploded extends Zip { return true; // TODO: hacky } - public hasEntry(entryPath: string): boolean { + public async hasEntry(entryPath: string): Promise { return this.hasEntries() && fs.existsSync(path.join(this.dirPath, entryPath)); } @@ -89,7 +91,7 @@ export class ZipExploded extends Zip { reset: async () => { return this.entryStreamPromise(entryPath); }, - stream: fs.createReadStream(fullPath, { autoClose: false }), + stream: toWebReadableStream(fs.createReadStream(fullPath, { autoClose: false })), }; return Promise.resolve(streamAndLength); diff --git a/src/_utils/zip/zip.ts b/src/_utils/zip/zip.ts index 9b4ec14..a25ed22 100644 --- a/src/_utils/zip/zip.ts +++ b/src/_utils/zip/zip.ts @@ -5,58 +5,62 @@ // that can be found in the LICENSE file exposed on Github (readium) in the project repository. // ==LICENSE-END== -import { RangeStream } from "../stream/RangeStream"; +// import { RangeStream } from "../stream/RangeStream"; export interface IStreamAndLength { - stream: NodeJS.ReadableStream; + stream: ReadableStream; length: number; reset: () => Promise; } +export interface IZipEntry { + stream(): Promise; + size(): Promise; +} + export interface IZip { hasEntries: () => boolean; entriesCount: () => number; - hasEntry: (entryPath: string) => boolean; + hasEntry: (entryPath: string) => Promise; getEntries: () => Promise; entryStreamPromise: (entryPath: string) => Promise; - entryStreamRangePromise: (entryPath: string, begin: number, end: number) => Promise; + // entryStreamRangePromise: (entryPath: string, begin: number, end: number) => Promise; freeDestroy: () => void; } export abstract class Zip implements IZip { public abstract hasEntries(): boolean; public abstract entriesCount(): number; - public abstract hasEntry(entryPath: string): boolean; + public abstract hasEntry(entryPath: string): Promise; public abstract getEntries(): Promise; public abstract entryStreamPromise(entryPath: string): Promise; public abstract freeDestroy(): void; - public async entryStreamRangePromise(entryPath: string, begin: number, end: number): Promise { - - let streamAndLength: IStreamAndLength; - try { - streamAndLength = await this.entryStreamPromise(entryPath); - } catch (err) { - console.log(err); - return Promise.reject(err); - } - - const b = begin < 0 ? 0 : begin; - const e = end < 0 ? (streamAndLength.length - 1) : end; - // const length = e - b + 1; - // debug(`entryStreamRangePromise: ${b}-${e}/${streamAndLength.length}`); - - const stream = new RangeStream(b, e, streamAndLength.length); - - streamAndLength.stream.pipe(stream); - - const sal: IStreamAndLength = { - length: streamAndLength.length, - reset: async () => { - return this.entryStreamRangePromise(entryPath, begin, end); - }, - stream, - }; - return sal; - } + // public async entryStreamRangePromise(entryPath: string, begin: number, end: number): Promise { + // + // let streamAndLength: IStreamAndLength; + // try { + // streamAndLength = await this.entryStreamPromise(entryPath); + // } catch (err) { + // console.log(err); + // return Promise.reject(err); + // } + // + // const b = begin < 0 ? 0 : begin; + // const e = end < 0 ? (streamAndLength.length - 1) : end; + // // const length = e - b + 1; + // // debug(`entryStreamRangePromise: ${b}-${e}/${streamAndLength.length}`); + // + // const stream = new RangeStream(b, e, streamAndLength.length); + // + // toNodeReadable(streamAndLength).stream.pipe(stream); + // + // return { + // length: streamAndLength.length, + // reset: async () => { + // return this.entryStreamRangePromise(entryPath, begin, end); + // }, + // stream: toWebReadableStream(stream), + // }; + // } } diff --git a/src/_utils/zip/zip1.ts b/src/_utils/zip/zip1.ts index b389ce9..3e09312 100644 --- a/src/_utils/zip/zip1.ts +++ b/src/_utils/zip/zip1.ts @@ -7,6 +7,7 @@ import * as debug_ from "debug"; import * as StreamZip from "node-stream-zip"; +import { toWebReadableStream } from "web-streams-node"; import { IStreamAndLength, IZip, Zip } from "./zip"; @@ -74,7 +75,7 @@ export class Zip1 extends Zip { return this.entriesCount() > 0; } - public hasEntry(entryPath: string): boolean { + public async hasEntry(entryPath: string): Promise { return this.hasEntries() && this.zip.entries()[entryPath]; } @@ -91,7 +92,7 @@ export class Zip1 extends Zip { // debug(`entryStreamPromise: ${entryPath}`); - if (!this.hasEntries() || !this.hasEntry(entryPath)) { + if (!this.hasEntries() || !await this.hasEntry(entryPath)) { return Promise.reject("no such path in zip: " + entryPath); } @@ -119,7 +120,7 @@ export class Zip1 extends Zip { reset: async () => { return this.entryStreamPromise(entryPath); }, - stream, + stream: toWebReadableStream(stream), }; resolve(streamAndLength); }); diff --git a/src/_utils/zip/zip2.ts b/src/_utils/zip/zip2.ts index eebf232..afcdb78 100644 --- a/src/_utils/zip/zip2.ts +++ b/src/_utils/zip/zip2.ts @@ -8,10 +8,11 @@ import * as debug_ from "debug"; import * as request from "request"; import * as requestPromise from "request-promise-native"; +import { toWebReadableStream } from "web-streams-node"; import * as yauzl from "yauzl"; import { isHTTP } from "../http/UrlUtils"; -import { streamToBufferPromise } from "../stream/BufferUtils"; +import { nodeStreamToBufferPromise } from "../stream/BufferUtils"; import { IStreamAndLength, IZip, Zip } from "./zip"; import { HttpZipReader } from "./zip2RandomAccessReader_Http"; @@ -129,7 +130,7 @@ export class Zip2 extends Zip { // debug(res.headers); let buffer: Buffer; try { - buffer = await streamToBufferPromise(ress); + buffer = await nodeStreamToBufferPromise(ress); } catch (err) { debug(err); reject(err); @@ -299,7 +300,7 @@ export class Zip2 extends Zip { return this.entriesCount() > 0; } - public hasEntry(entryPath: string): boolean { + public async hasEntry(entryPath: string): Promise { return this.hasEntries() && this.entries[entryPath]; } @@ -315,7 +316,7 @@ export class Zip2 extends Zip { // debug(`entryStreamPromise: ${entryPath}`); - if (!this.hasEntries() || !this.hasEntry(entryPath)) { + if (!this.hasEntries() || !await this.hasEntry(entryPath)) { return Promise.reject("no such path in zip: " + entryPath); } @@ -335,7 +336,7 @@ export class Zip2 extends Zip { reset: async () => { return this.entryStreamPromise(entryPath); }, - stream, + stream: toWebReadableStream(stream), }; resolve(streamAndLength); }); diff --git a/src/_utils/zip/zip2RandomAccessReader_Http.ts b/src/_utils/zip/zip2RandomAccessReader_Http.ts index c8e68cd..b0c65af 100644 --- a/src/_utils/zip/zip2RandomAccessReader_Http.ts +++ b/src/_utils/zip/zip2RandomAccessReader_Http.ts @@ -13,7 +13,7 @@ import * as request from "request"; import * as requestPromise from "request-promise-native"; import * as yauzl from "yauzl"; -import { bufferToStream, streamToBufferPromise } from "../stream/BufferUtils"; +import { bufferToNodeStream, nodeStreamToBufferPromise } from "../stream/BufferUtils"; // import { HttpReadableStream } from "./HttpReadableStream"; @@ -52,7 +52,7 @@ export class HttpZipReader implements RandomAccessReader { const begin = start - this.firstBufferStart; const stop = end - this.firstBufferStart; - return bufferToStream(this.firstBuffer.slice(begin, stop)); + return bufferToNodeStream(this.firstBuffer.slice(begin, stop)); } const stream = new PassThrough(); @@ -88,7 +88,7 @@ export class HttpZipReader implements RandomAccessReader { } else { let buffer: Buffer; try { - buffer = await streamToBufferPromise(res); + buffer = await nodeStreamToBufferPromise(res); } catch (err) { debug(err); stream.end(); diff --git a/src/_utils/zip/zip3.ts b/src/_utils/zip/zip3.ts index e29361a..110c3c2 100644 --- a/src/_utils/zip/zip3.ts +++ b/src/_utils/zip/zip3.ts @@ -9,6 +9,7 @@ import * as debug_ from "debug"; import * as request from "request"; import * as unzipper from "unzipper"; +import { toWebReadableStream } from "web-streams-node"; import { isHTTP } from "../http/UrlUtils"; import { IStreamAndLength, IZip, Zip } from "./zip"; @@ -86,7 +87,7 @@ export class Zip3 extends Zip { return this.entriesCount() > 0; } - public hasEntry(entryPath: string): boolean { + public async hasEntry(entryPath: string): Promise { return this.hasEntries() && this.entries[entryPath]; } @@ -117,7 +118,7 @@ export class Zip3 extends Zip { reset: async () => { return this.entryStreamPromise(entryPath); }, - stream, + stream: toWebReadableStream(stream), }; resolve(streamAndLength); }); diff --git a/src/_utils/zip/zip4.ts b/src/_utils/zip/zip4.ts new file mode 100644 index 0000000..5019ceb --- /dev/null +++ b/src/_utils/zip/zip4.ts @@ -0,0 +1,94 @@ +// import * as debug_ from "debug"; +// import { lstat } from "fs"; +// import * as globAsync from "glob"; +// import * as path from "path"; +// import { promisify } from "util"; +// +// const fileStat = promisify(lstat); +// const glob = promisify(globAsync); +// +// import { IStreamAndLength, IZip, IZipEntry, Zip } from "./zip"; +// import { Zip4Entry } from "./zip4Entry"; +// +// const debug = debug_("r2:utils#zip/zip4"); +// +// export class Zip4 extends Zip { +// +// public static async loadPromise(filePath: string): Promise { +// return new Promise(async (resolve, reject) => { +// if ((await fileStat(filePath)).isFile()) { +// return reject("not a directory, expected a path to an exploded (uncompressed) zip"); +// } +// +// const zip4 = new Zip4(filePath); +// +// const filesInPath = await glob("**/*", { cwd: filePath, nodir: true }); +// filesInPath.forEach((file: string) => { +// zip4.entries.set(file, new Zip4Entry(path.join(filePath, file))); +// }); +// +// resolve(zip4); +// }); +// } +// +// private entries: Map; +// +// private constructor(readonly filePath: string) { +// super(); +// +// this.entries = new Map(); +// } +// +// public freeDestroy(): void { +// debug("freeDestroy: Zip4 -- " + this.filePath); +// } +// +// public entriesCount(): number { +// return this.entries.size; +// } +// +// public hasEntries(): boolean { +// return this.entriesCount() > 0; +// } +// +// public async hasEntry(entryPath: string): Promise { +// return this.hasEntries() && !!this.entries.get(entryPath); +// } +// +// public forEachEntry(callback: (entryName: string) => void) { +// +// if (!this.hasEntries()) { +// return; +// } +// +// Object.keys(this.entries).forEach((entryName) => { +// callback(entryName); +// }); +// } +// +// public async entryStreamPromise(entryPath: string): Promise { +// if (!this.hasEntries()) { +// return Promise.reject("zip is empty"); +// } +// +// // debug(`entryStreamPromise: ${entryPath}`); +// const entry = this.entries.get(entryPath); +// +// if (!entry) { +// return Promise.reject("no such path in zip: " + entryPath); +// } +// +// return new Promise(async (resolve, _reject) => { +// debug(entry); +// +// const streamAndLength: IStreamAndLength = { +// length: await entry.size(), +// reset: async () => { +// return this.entryStreamPromise(entryPath); +// }, +// stream: await entry.stream(), +// }; +// resolve(streamAndLength); +// }); +// } +// } diff --git a/src/_utils/zip/zip4Entry.ts b/src/_utils/zip/zip4Entry.ts new file mode 100644 index 0000000..a422f4f --- /dev/null +++ b/src/_utils/zip/zip4Entry.ts @@ -0,0 +1,19 @@ +// import { createReadStream, stat } from "fs"; +// import { promisify } from "util"; +// import { toWebReadableStream } from "web-streams-node"; +// import { IZipEntry } from "./zip"; +// +// const fsStat = promisify(stat); +// +// export class Zip4Entry implements IZipEntry { +// constructor(private path: string) { +// } +// +// public async stream(): Promise { +// return toWebReadableStream(createReadStream(this.path)); +// } +// +// public async size(): Promise { +// return (await fsStat(this.path)).size; +// } +// } diff --git a/src/_utils/zip/zip5.ts b/src/_utils/zip/zip5.ts new file mode 100644 index 0000000..a8ff1cc --- /dev/null +++ b/src/_utils/zip/zip5.ts @@ -0,0 +1,84 @@ +import * as debug_ from "debug"; +import { URL } from "isomorphic-url-shim"; +import { IStreamAndLength, IZip, Zip } from "./zip"; +import { Zip5Entry } from "./zip5Entry"; + +const debug = debug_("r2:utils#zip/zip5"); + +export class Zip5 extends Zip { + + public static async loadPromise(url: string): Promise { + return new Promise(async (resolve, _reject) => { + const zip5 = new Zip5(url); + const testFile = "META-INF/container.xml"; + const urlString = new URL(testFile, url); + const entry = new Zip5Entry(urlString.toString()); + zip5.entries.set(testFile, entry); + resolve(zip5); + }); + } + + private baseURL: string; + private entries: Map; + + private constructor(readonly filePath: string) { + super(); + this.baseURL = filePath; + this.entries = new Map(); + } + + public freeDestroy(): void { + debug("freeDestroy: Zip5 -- " + this.filePath); + } + + public entriesCount(): number { + return this.entries.size; + } + + public hasEntries(): boolean { + return this.entriesCount() > 0; + } + + public async hasEntry(entryPath: string): Promise { + return this.getEntry(entryPath).exists(); + } + + public async getEntries(): Promise { + + if (!this.hasEntries()) { + return Promise.resolve([]); + } + return Promise.resolve(Object.keys(this.entries)); + } + + public async entryStreamPromise(entryPath: string): Promise { + const entry = this.getEntry(entryPath); + + if (!entry.exists()) { + return Promise.reject("no such path in zip: " + entryPath); + } + + return new Promise(async (resolve, _reject) => { + debug(entry); + + const streamAndLength: IStreamAndLength = { + length: await entry.size(), + reset: async () => { + return this.entryStreamPromise(entryPath); + }, + stream: await entry.stream(), + }; + resolve(streamAndLength); + }); + } + + private getEntry(entryPath: string) { + const entry = this.entries.get(entryPath); + if (!entry) { + const newEntry = new Zip5Entry(new URL(entryPath, this.baseURL).toString()); + this.entries.set(entryPath, newEntry); + return newEntry; + } + return entry; + } +} diff --git a/src/_utils/zip/zip5Entry.ts b/src/_utils/zip/zip5Entry.ts new file mode 100644 index 0000000..ab6575c --- /dev/null +++ b/src/_utils/zip/zip5Entry.ts @@ -0,0 +1,27 @@ +import crossFetch from "cross-fetch"; +import fetchStream from "isomorphic-fetch-readablestream"; + +import { IZipEntry } from "./zip"; + +const fetch = self.fetch || crossFetch; + +export class Zip5Entry implements IZipEntry { + private fetchHead: Promise; + + constructor(private path: string) { + this.fetchHead = fetch(this.path, { method: "HEAD" }); + } + + public async stream(): Promise { + const response = await fetchStream(this.path); + return response.body; + } + + public async size(): Promise { + return parseInt((await this.fetchHead).headers.get("content-length") || "0", 10); + } + + public async exists(): Promise { + return (await this.fetchHead).ok; + } +} diff --git a/src/_utils/zip/zipFactory.ts b/src/_utils/zip/zipFactory.ts index 8dd0318..d62d0c9 100644 --- a/src/_utils/zip/zipFactory.ts +++ b/src/_utils/zip/zipFactory.ts @@ -5,23 +5,46 @@ // that can be found in the LICENSE file exposed on Github (readium) in the project repository. // ==LICENSE-END== -import * as fs from "fs"; +// import * as fs from "fs"; -import { isHTTP } from "../http/UrlUtils"; +import { isDirectory, isHTTP } from "../http/UrlUtils"; import { IZip } from "./zip"; -import { ZipExploded } from "./zip-ex"; -import { Zip1 } from "./zip1"; -import { Zip2 } from "./zip2"; +// import { ZipExploded } from "./zip-ex"; +// import { Zip1 } from "./zip1"; +// import { Zip2 } from "./zip2"; +import { Zip5 } from "./zip5"; + +// export async function zipLoadPromise(filePath: string): Promise { +// if (isHTTP(filePath)) { +// return Zip2.loadPromise(filePath); +// } +// +// const stats = fs.lstatSync(filePath); +// if (stats.isDirectory()) { +// return ZipExploded.loadPromise(filePath); +// } +// +// return Zip1.loadPromise(filePath); +// } export async function zipLoadPromise(filePath: string): Promise { - if (isHTTP(filePath)) { - return Zip2.loadPromise(filePath); - } + const http = isHTTP(filePath); + const dir = isDirectory(filePath); - const stats = fs.lstatSync(filePath); - if (stats.isDirectory()) { - return ZipExploded.loadPromise(filePath); + if (http && !dir) { + // HTTP url to a zip file + // return Zip2.loadPromise(filePath); + throw new Error("not implemented"); + } else if (!dir) { + // Path to a zip file + // return Zip1.loadPromise(filePath); + throw new Error("not implemented"); + } else if (http) { + // HTTP url to an exploded (uncompressed) directory + return Zip5.loadPromise(filePath); } - return Zip1.loadPromise(filePath); + // Path to an exploded (uncompressed) directory + // return ZipExploded.loadPromise(filePath); + throw new Error("not implemented"); } diff --git a/src/declarations.d.ts b/src/declarations.d.ts index 52d8df5..073ed2a 100644 --- a/src/declarations.d.ts +++ b/src/declarations.d.ts @@ -7,6 +7,8 @@ // declare module "*"; +declare module "cross-fetch"; +declare module "fetch-readablestream"; declare module "filehound"; declare module "node-stream-zip"; declare module "unzipper"; diff --git a/tsconfigs/tsconfig-common.json b/tsconfigs/tsconfig-common.json index 76ab6e4..53743d3 100644 --- a/tsconfigs/tsconfig-common.json +++ b/tsconfigs/tsconfig-common.json @@ -23,6 +23,9 @@ ], "@utils/*": [ "src/_utils/*" + ], + "@ta-json/*": [ + "node_modules/ta-json/dist/es6/*" ] }, "noImplicitAny": true, @@ -71,7 +74,8 @@ "preserveConstEnums": false, "removeComments": true, "skipLibCheck": false, - "locale": "en-US" + "locale": "en-US", + "types": [] }, "exclude": [ "node_modules"