From 3b3112047ba3b74b700d13d1587fb82c1170ce6e Mon Sep 17 00:00:00 2001 From: rabinzon Date: Mon, 11 Dec 2017 09:15:26 +0300 Subject: [PATCH] feat: add lister tasks --- package-lock.json | 214 ++++++++++++++++++++++++++++------------- package.json | 1 + src/bin/page-loader.js | 2 +- src/index.js | 60 ++++++------ 4 files changed, 175 insertions(+), 102 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4a124e0..368c25e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -99,14 +99,17 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "dev": true + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "any-observable": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/any-observable/-/any-observable-0.2.0.tgz", + "integrity": "sha1-xnhwBYADV5AJCD9UrAq6+1wz0kI=" }, "any-promise": { "version": "1.3.0", @@ -1522,7 +1525,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "dev": true, "requires": { "ansi-styles": "2.2.1", "escape-string-regexp": "1.0.5", @@ -1596,14 +1598,12 @@ "cli-spinners": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-0.1.2.tgz", - "integrity": "sha1-u3ZNiOGF+54eaiofGXcjGPYF4xw=", - "dev": true + "integrity": "sha1-u3ZNiOGF+54eaiofGXcjGPYF4xw=" }, "cli-truncate": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-0.2.1.tgz", "integrity": "sha1-nxXPuwcFAFNpIWxiasfQWrkN1XQ=", - "dev": true, "requires": { "slice-ansi": "0.0.4", "string-width": "1.0.2" @@ -1613,7 +1613,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, "requires": { "number-is-nan": "1.0.1" } @@ -1622,7 +1621,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, "requires": { "code-point-at": "1.1.0", "is-fullwidth-code-point": "1.0.0", @@ -1667,8 +1665,7 @@ "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, "color-convert": { "version": "1.9.0", @@ -2107,8 +2104,7 @@ "date-fns": { "version": "1.29.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz", - "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==", - "dev": true + "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==" }, "dateformat": { "version": "1.0.12", @@ -2301,8 +2297,7 @@ "elegant-spinner": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/elegant-spinner/-/elegant-spinner-1.0.1.tgz", - "integrity": "sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4=", - "dev": true + "integrity": "sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4=" }, "entities": { "version": "1.1.1", @@ -2330,8 +2325,7 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "escodegen": { "version": "1.9.0", @@ -2649,8 +2643,7 @@ "exit-hook": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz", - "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g=", - "dev": true + "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g=" }, "expand-brackets": { "version": "0.1.5", @@ -4010,7 +4003,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "dev": true, "requires": { "ansi-regex": "2.1.1" } @@ -4115,7 +4107,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", - "dev": true, "requires": { "repeating": "2.0.1" } @@ -4294,7 +4285,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=", - "dev": true, "requires": { "number-is-nan": "1.0.1" } @@ -4329,6 +4319,21 @@ "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", "dev": true }, + "is-observable": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-0.2.0.tgz", + "integrity": "sha1-s2ExHYPG5dcmyr9eJQsCNxBvWuI=", + "requires": { + "symbol-observable": "0.2.4" + }, + "dependencies": { + "symbol-observable": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-0.2.4.tgz", + "integrity": "sha1-lag9smGG1q9+ehjb2XYKL4bQj0A=" + } + } + }, "is-path-cwd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", @@ -4368,8 +4373,7 @@ "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", - "dev": true + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" }, "is-regexp": { "version": "1.0.0", @@ -4389,8 +4393,7 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", - "dev": true + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, "is-subset": { "version": "0.1.1", @@ -5765,6 +5768,16 @@ "supports-color": "4.4.0" } }, + "figures": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz", + "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=", + "dev": true, + "requires": { + "escape-string-regexp": "1.0.5", + "object-assign": "4.1.1" + } + }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -5780,6 +5793,66 @@ "is-extglob": "2.1.1" } }, + "listr": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/listr/-/listr-0.12.0.tgz", + "integrity": "sha1-a84sD1YD+klYDqF81qAMwOX6RRo=", + "dev": true, + "requires": { + "chalk": "1.1.3", + "cli-truncate": "0.2.1", + "figures": "1.7.0", + "indent-string": "2.1.0", + "is-promise": "2.1.0", + "is-stream": "1.1.0", + "listr-silent-renderer": "1.1.1", + "listr-update-renderer": "0.2.0", + "listr-verbose-renderer": "0.4.0", + "log-symbols": "1.0.2", + "log-update": "1.0.2", + "ora": "0.2.3", + "p-map": "1.2.0", + "rxjs": "5.4.3", + "stream-to-observable": "0.1.0", + "strip-ansi": "3.0.1" + }, + "dependencies": { + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "2.2.1", + "escape-string-regexp": "1.0.5", + "has-ansi": "2.0.0", + "strip-ansi": "3.0.1", + "supports-color": "2.0.0" + } + }, + "log-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz", + "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=", + "dev": true, + "requires": { + "chalk": "1.1.3" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + } + } + }, "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", @@ -5792,26 +5865,26 @@ } }, "listr": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/listr/-/listr-0.12.0.tgz", - "integrity": "sha1-a84sD1YD+klYDqF81qAMwOX6RRo=", - "dev": true, + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/listr/-/listr-0.13.0.tgz", + "integrity": "sha1-ILsLowuuZg7oTMBQPfS+PVYjiH0=", "requires": { "chalk": "1.1.3", "cli-truncate": "0.2.1", "figures": "1.7.0", "indent-string": "2.1.0", + "is-observable": "0.2.0", "is-promise": "2.1.0", "is-stream": "1.1.0", "listr-silent-renderer": "1.1.1", - "listr-update-renderer": "0.2.0", + "listr-update-renderer": "0.4.0", "listr-verbose-renderer": "0.4.0", "log-symbols": "1.0.2", "log-update": "1.0.2", "ora": "0.2.3", "p-map": "1.2.0", "rxjs": "5.4.3", - "stream-to-observable": "0.1.0", + "stream-to-observable": "0.2.0", "strip-ansi": "3.0.1" }, "dependencies": { @@ -5819,28 +5892,55 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz", "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=", - "dev": true, "requires": { "escape-string-regexp": "1.0.5", "object-assign": "4.1.1" } }, + "listr-update-renderer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/listr-update-renderer/-/listr-update-renderer-0.4.0.tgz", + "integrity": "sha1-NE2YDaLKLosUW6MFkI8yrj9MyKc=", + "requires": { + "chalk": "1.1.3", + "cli-truncate": "0.2.1", + "elegant-spinner": "1.0.1", + "figures": "1.7.0", + "indent-string": "3.2.0", + "log-symbols": "1.0.2", + "log-update": "1.0.2", + "strip-ansi": "3.0.1" + }, + "dependencies": { + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" + } + } + }, "log-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz", "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=", - "dev": true, "requires": { "chalk": "1.1.3" } + }, + "stream-to-observable": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/stream-to-observable/-/stream-to-observable-0.2.0.tgz", + "integrity": "sha1-WdbqOT2HwsDdrBCqDVYbxrpvDhA=", + "requires": { + "any-observable": "0.2.0" + } } } }, "listr-silent-renderer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz", - "integrity": "sha1-kktaN1cVN3C/Go4/v3S4u/P5JC4=", - "dev": true + "integrity": "sha1-kktaN1cVN3C/Go4/v3S4u/P5JC4=" }, "listr-update-renderer": { "version": "0.2.0", @@ -5889,7 +5989,6 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/listr-verbose-renderer/-/listr-verbose-renderer-0.4.0.tgz", "integrity": "sha1-RNwBuww0oDxXIVTU0Izemx3FYg8=", - "dev": true, "requires": { "chalk": "1.1.3", "cli-cursor": "1.0.2", @@ -5901,7 +6000,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz", "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=", - "dev": true, "requires": { "restore-cursor": "1.0.1" } @@ -5910,7 +6008,6 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz", "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=", - "dev": true, "requires": { "escape-string-regexp": "1.0.5", "object-assign": "4.1.1" @@ -5919,14 +6016,12 @@ "onetime": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", - "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=", - "dev": true + "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "restore-cursor": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz", "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=", - "dev": true, "requires": { "exit-hook": "1.1.1", "onetime": "1.1.0" @@ -6044,7 +6139,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/log-update/-/log-update-1.0.2.tgz", "integrity": "sha1-GZKfZMQJPS0ucHWh2tivWcKWuNE=", - "dev": true, "requires": { "ansi-escapes": "1.4.0", "cli-cursor": "1.0.2" @@ -6053,14 +6147,12 @@ "ansi-escapes": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz", - "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4=", - "dev": true + "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4=" }, "cli-cursor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz", "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=", - "dev": true, "requires": { "restore-cursor": "1.0.1" } @@ -6068,14 +6160,12 @@ "onetime": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", - "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=", - "dev": true + "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "restore-cursor": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz", "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=", - "dev": true, "requires": { "exit-hook": "1.1.1", "onetime": "1.1.0" @@ -6443,8 +6533,7 @@ "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, "nwmatcher": { "version": "1.4.3", @@ -6527,7 +6616,6 @@ "version": "0.2.3", "resolved": "https://registry.npmjs.org/ora/-/ora-0.2.3.tgz", "integrity": "sha1-N1J9Igrc1Tw5tzVx11QVbV22V6Q=", - "dev": true, "requires": { "chalk": "1.1.3", "cli-cursor": "1.0.2", @@ -6539,7 +6627,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz", "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=", - "dev": true, "requires": { "restore-cursor": "1.0.1" } @@ -6547,14 +6634,12 @@ "onetime": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", - "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=", - "dev": true + "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "restore-cursor": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz", "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=", - "dev": true, "requires": { "exit-hook": "1.1.1", "onetime": "1.1.0" @@ -6649,8 +6734,7 @@ "p-map": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz", - "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==", - "dev": true + "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==" }, "parse-github-repo-url": { "version": "1.4.1", @@ -7079,7 +7163,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", - "dev": true, "requires": { "is-finite": "1.0.2" } @@ -7214,7 +7297,6 @@ "version": "5.4.3", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.4.3.tgz", "integrity": "sha512-fSNi+y+P9ss+EZuV0GcIIqPUK07DEaMRUtLJvdcvMyFjc9dizuDjere+A4V7JrLGnm9iCc+nagV/4QdMTkqC4A==", - "dev": true, "requires": { "symbol-observable": "1.0.4" } @@ -7308,8 +7390,7 @@ "slice-ansi": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz", - "integrity": "sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU=", - "dev": true + "integrity": "sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU=" }, "sntp": { "version": "2.0.2", @@ -7528,7 +7609,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, "requires": { "ansi-regex": "2.1.1" } @@ -7563,14 +7643,12 @@ "supports-color": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, "symbol-observable": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.4.tgz", - "integrity": "sha1-Kb9hXUqnEhvdiYsi1LP5vE4qoD0=", - "dev": true + "integrity": "sha1-Kb9hXUqnEhvdiYsi1LP5vE4qoD0=" }, "symbol-tree": { "version": "3.2.2", diff --git a/package.json b/package.json index 1a0c556..a93784b 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "debug": "^3.1.0", "file-type": "^7.4.0", "is-svg": "^2.1.0", + "listr": "^0.13.0", "mz": "^2.7.0", "nock": "^9.1.4" } diff --git a/src/bin/page-loader.js b/src/bin/page-loader.js index ecf3420..ed30ad1 100644 --- a/src/bin/page-loader.js +++ b/src/bin/page-loader.js @@ -13,7 +13,7 @@ program if (!url) { program.help(); } - pageLoader(url, options.ouatput) + pageLoader(url, options.output) .then((fileName) => { console.log(`Page was downloaded as '${fileName}'`); process.exit(0); diff --git a/src/index.js b/src/index.js index f129310..a7e12d8 100644 --- a/src/index.js +++ b/src/index.js @@ -3,6 +3,7 @@ import fs from 'mz/fs'; import debug from 'debug'; import path from 'path'; import process from 'process'; +import Listr from 'listr'; import Url from 'url'; import cheerio from 'cheerio'; @@ -13,7 +14,8 @@ const getFileName = (src) => { const { host, pathname } = Url.parse(src); const fileName = `${host || ''}${pathname}`.replace(/\.[a-z]+$/g, '').replace(/\W/gi, '-'); const ext = path.extname(pathname); - return ext ? `${fileName}${ext}` : fileName; + const newName = ext ? `${fileName}${ext}` : fileName; + return newName.replace(/(^-|-$)/, ''); }; const fetch = (url, params = {}) => @@ -24,49 +26,41 @@ const fetch = (url, params = {}) => }) .catch(error => Promise.reject(new Error(`GET ${url} ${error.message}`))); -const loadResources = (html) => { +const loadResources = (url, outputPath) => (html) => { const $ = cheerio.load(html); - const resourceTags = ['link', 'script', 'img']; - const htmlTags = [...$(resourceTags.map(tag => `${tag}[src]`).toString())]; + const htmlTags = [...$('link[src], script[src], img[src]')]; + const filesDirName = `${getFileName(url)}_files`; + const filesDirPath = path.resolve(outputPath, filesDirName); const fetchResource = src => fetch(src, { responseType: 'arraybuffer' }) .then(file => ({ src, file, newName: getFileName(src) })); - return Promise.all(htmlTags - .map(tag => fetchResource(tag.attribs.src))) - .then(files => ({ html, files })); + return fs.exists(filesDirPath) + .then(exists => (exists ? filesDirPath : fs.mkdir(filesDirPath))) + .then(() => new Listr(htmlTags + .map(({ attribs: { src } }) => ( + { title: src, + task: () => fetchResource(src) + .then(({ newName, file }) => fs.writeFile(path.resolve(filesDirPath, newName), file, 'binary')) + .then(() => { + debugWriteFile('recourse saved:', getFileName(src)); + $(`[src='${src}']`).attr('src', `${filesDirName}/${getFileName(src)}`); + }), + } + )), { concurrent: true }).run()) + .then(() => $.html()); }; -const writeToFile = ({ html, files }, url, outputPath) => { - const $ = cheerio.load(html); - const pageFileName = getFileName(url); - const outputDir = path.resolve(outputPath, `${pageFileName}.html`); - const filesDir = path.resolve(outputPath, `${pageFileName}_files`); - - files.forEach(({ src, newName }) => - $(`[src='${src}']`).attr('src', `${pageFileName}_files/${newName}`)); - - debugWriteFile('output path:', outputDir); - - return fs.exists(filesDir) - .then(exists => (exists ? filesDir : fs.mkdir(filesDir))) - .then(() => fs.writeFile(outputDir, $.html())) - .then(() => - Promise.all(files.map(({ file, newName }) => { - debugWriteFile('writing files to:', filesDir); - return fs.writeFile(path.resolve(filesDir, newName), file, 'binary'); - }))) - .then(() => `${pageFileName}.html`) - .catch(err => Promise.reject(new Error(err.message))); -}; - -export default (url, outputPath = process.pwd()) => +export default (url, outputPath = process.cwd()) => fs.exists(outputPath).then((exists) => { if (!exists) { return Promise.reject(new Error(`${outputPath} directory does not exist`)); } + const fileName = `${getFileName(url)}.html`; + const outputDir = path.resolve(outputPath, fileName); return fetch(url) - .then(loadResources) - .then(page => writeToFile(page, url, outputPath)); + .then(loadResources(url, outputPath)) + .then(html => fs.writeFile(outputDir, html)) + .then(() => fileName); });