diff --git a/index.js b/index.js index 57fa19f..dd2cf0d 100644 --- a/index.js +++ b/index.js @@ -11,12 +11,13 @@ function initServer() { app.listen(config.port, () => { console.info(`server started on port ${config.port} (${config.env})`); // eslint-disable-line no-console }); + ingest() } } function ingest() { const sedTranscriptIngester = new SEDTranscriptIngester(); - sedTranscriptIngester.getContent() + sedTranscriptIngester.getContentLatest() } switch(process.argv[2]) { diff --git a/package-lock.json b/package-lock.json index b7da5a0..0779773 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,7 @@ "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "integrity": "sha1-6D0X3hbYp++3cX7b5fsQE17uYps=", "dev": true, "requires": { "ms": "^2.1.1" @@ -47,7 +47,7 @@ "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "integrity": "sha1-MKWGTrPrsKZvLr5tcnrwagnYbgo=", "dev": true } } @@ -722,7 +722,7 @@ "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "integrity": "sha1-6D0X3hbYp++3cX7b5fsQE17uYps=", "dev": true, "requires": { "ms": "^2.1.1" @@ -731,7 +731,7 @@ "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "integrity": "sha1-MKWGTrPrsKZvLr5tcnrwagnYbgo=", "dev": true } } @@ -778,6 +778,11 @@ "through2": "^2.0.3" } }, + "abab": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz", + "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -799,6 +804,38 @@ "integrity": "sha1-Z6ojG/iBKXS4UjWpZ3Hra9B+onk=", "dev": true }, + "acorn-globals": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.0.tgz", + "integrity": "sha512-hMtHj3s5RnuhvHPowpBYvJVj3rAar82JiDQHvGs1zO0l10ocX/xEdBShNHTJaboucJUsScghp74pH3s7EnHHQw==", + "requires": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + }, + "dependencies": { + "acorn": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.4.tgz", + "integrity": "sha512-VY4i5EKSKkofY2I+6QLTbTTN/UvEQPCo6eiwzzSaSWfpaDhOmStMCMod6wmuPciNq+XS0faCglFu2lHZpdHUtg==" + } + } + }, + "acorn-walk": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.1.0.tgz", + "integrity": "sha512-ugTb7Lq7u4GfWSqqpwE0bGyoBZNMTok/zDBXxfEG0QM50jNlGhIWjRC1pPN7bvV1anhF+bs+/gNcRw+o55Evbg==" + }, + "ajv": { + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.5.tgz", + "integrity": "sha512-7q7gtRQDJSyuEHjuVgHoUa2VuemFiCMrfQc9Tc08XTAc4Zj/5U1buQJ0HU6i7fKjXU09SVgSmxa4sLvuvS8Iyg==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -965,6 +1002,11 @@ "integrity": "sha1-p5SvDAWrF1KEbudTofIRoFugxE8=", "dev": true }, + "array-equal": { + "version": "1.0.0", + "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1099,6 +1141,11 @@ "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0=", "dev": true }, + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + }, "async-settle": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz", @@ -1108,6 +1155,11 @@ "async-done": "^1.2.2" } }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", @@ -1120,6 +1172,11 @@ "integrity": "sha1-xXED96F/wDfwLXwuZLYC6iI/fWM=", "optional": true }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, "bach": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/bach/-/bach-1.2.0.tgz", @@ -1197,12 +1254,25 @@ } } }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, "binary-extensions": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz", "integrity": "sha1-wteA9T1Fu6gxeokC1M7q86Y4WxQ=", "dev": true }, + "bluebird": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", + "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" + }, "body-parser": { "version": "1.18.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", @@ -1224,7 +1294,6 @@ "version": "0.4.2", "resolved": "http://registry.npmjs.org/boom/-/boom-0.4.2.tgz", "integrity": "sha1-emNune1O/O+xnO9JR6PGffrukRs=", - "optional": true, "requires": { "hoek": "0.9.x" }, @@ -1232,8 +1301,7 @@ "hoek": { "version": "0.9.1", "resolved": "http://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz", - "integrity": "sha1-PTIkYrrfB3Fup+uFuviAec3c5QU=", - "optional": true + "integrity": "sha1-PTIkYrrfB3Fup+uFuviAec3c5QU=" } } }, @@ -1335,6 +1403,11 @@ } } }, + "browser-process-hrtime": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz", + "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==" + }, "browserslist": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.3.4.tgz", @@ -1404,6 +1477,11 @@ "integrity": "sha1-psC74fOPOqC5Ijjstv9Cw0TUE10=", "dev": true }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, "chalk": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", @@ -1659,8 +1737,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cors": { "version": "2.8.5", @@ -1726,6 +1803,19 @@ } } }, + "cssom": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.4.tgz", + "integrity": "sha512-+7prCSORpXNeR4/fUP3rL+TzqtiFfhMvTd7uEqMdgPvLPt4+uzFUeufx5RHjGTACCargg/DiEt/moMQmvnfkog==" + }, + "cssstyle": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.1.1.tgz", + "integrity": "sha512-364AI1l/M5TYcFH83JnOH/pSqgaNnKmYgKrm0didZMGKWjQB60dymwWy1rKUgL3J1ffdq9xVi2yGLHdSjjSNog==", + "requires": { + "cssom": "0.3.x" + } + }, "ctype": { "version": "0.5.3", "resolved": "http://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz", @@ -1741,6 +1831,31 @@ "es5-ext": "^0.10.9" } }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "requires": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -1795,6 +1910,11 @@ "integrity": "sha1-xPp8lUBKF6nD6Mp+FTcxK3NjMKw=", "dev": true }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, "default-compare": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/default-compare/-/default-compare-1.0.0.tgz", @@ -1916,6 +2036,14 @@ "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=", "dev": true }, + "domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "requires": { + "webidl-conversions": "^4.0.2" + } + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -1964,6 +2092,15 @@ "object.defaults": "^1.1.0" } }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -2053,11 +2190,40 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "escodegen": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.0.tgz", + "integrity": "sha512-IeMV45ReixHS53K/OmfKAIztN/igDHzTJUhZM3k1jMhIZWjk45SMwAtBsEXiJp3vSPmTcu6CXn7mDvFHRN66fw==", + "requires": { + "esprima": "^3.1.3", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true + } + } + }, + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + }, + "estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" + }, "esutils": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", - "dev": true + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" }, "etag": { "version": "1.8.1", @@ -2246,8 +2412,7 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=", - "dev": true + "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=" }, "extend-shallow": { "version": "3.0.2", @@ -2330,6 +2495,11 @@ } } }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, "fancy-log": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fancy-log/-/fancy-log-1.3.2.tgz", @@ -2341,6 +2511,21 @@ "time-stamp": "^1.0.0" } }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, "filename-regex": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz", @@ -3096,6 +3281,21 @@ "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", "dev": true }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, "glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", @@ -3626,6 +3826,20 @@ "request": "~2.36.0" } }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -3719,6 +3933,14 @@ "integrity": "sha1-l/I2l3vW4SVAiTD/bePuxigewEc=", "dev": true }, + "html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "requires": { + "whatwg-encoding": "^1.0.1" + } + }, "http-errors": { "version": "1.6.3", "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -4097,6 +4319,11 @@ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", "dev": true }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, "is-unc-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz", @@ -4150,6 +4377,11 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, "joi": { "version": "14.0.4", "resolved": "https://registry.npmjs.org/joi/-/joi-14.0.4.tgz", @@ -4172,12 +4404,160 @@ "integrity": "sha1-GSA/tZmR35jjoocFDUZHzerzJJk=", "dev": true }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsdom": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-13.0.0.tgz", + "integrity": "sha512-Kmq4ASMNkgpY+YufE322EnIKoiz0UWY2DRkKlU7d5YrIW4xiVRhWFrZV1fr6w/ZNxQ50wGAH5gGRzydgnmkkvw==", + "requires": { + "abab": "^2.0.0", + "acorn": "^6.0.2", + "acorn-globals": "^4.3.0", + "array-equal": "^1.0.0", + "cssom": "^0.3.4", + "cssstyle": "^1.1.1", + "data-urls": "^1.0.1", + "domexception": "^1.0.1", + "escodegen": "^1.11.0", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.0.9", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.5", + "saxes": "^3.1.3", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.4.3", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.0.0", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0", + "ws": "^6.1.0", + "xml-name-validator": "^3.0.0" + }, + "dependencies": { + "acorn": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.4.tgz", + "integrity": "sha512-VY4i5EKSKkofY2I+6QLTbTTN/UvEQPCo6eiwzzSaSWfpaDhOmStMCMod6wmuPciNq+XS0faCglFu2lHZpdHUtg==" + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "jsesc": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.1.tgz", "integrity": "sha1-5CGiqOINawgZ3yiQj3glJrlt0f4=", "dev": true }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "json-stable-stringify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", @@ -4204,6 +4584,24 @@ "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=", "dev": true }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, "just-debounce": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/just-debounce/-/just-debounce-1.0.0.tgz", @@ -4268,6 +4666,15 @@ "flush-write-stream": "^1.0.2" } }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, "liftoff": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/liftoff/-/liftoff-2.5.0.tgz", @@ -4300,8 +4707,7 @@ "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha1-s56mIp72B+zYniyN8SU2iRysm40=", - "dev": true + "integrity": "sha1-s56mIp72B+zYniyN8SU2iRysm40=" }, "lodash.debounce": { "version": "4.0.8", @@ -4309,6 +4715,11 @@ "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", "dev": true }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -4658,6 +5069,11 @@ "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "dev": true }, + "nwsapi": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.0.9.tgz", + "integrity": "sha512-nlWFSCTYQcHk/6A9FFnfhKc14c3aFhfdNBXgo8Qgi9QTBu/qg3Ww+Uiz9wMzXd1T8GFxPc2QIHB6Qtf2XFryFQ==" + }, "oauth-sign": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.3.0.tgz", @@ -4811,6 +5227,19 @@ "wrappy": "1" } }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + } + }, "ordered-read-streams": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ordered-read-streams/-/ordered-read-streams-1.0.1.tgz", @@ -4913,6 +5342,11 @@ "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", "dev": true }, + "parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" + }, "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", @@ -5025,6 +5459,11 @@ } } }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, "pify": { "version": "2.3.0", "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -5055,12 +5494,22 @@ "extend-shallow": "^3.0.2" } }, + "pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, "prepend-http": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", @@ -5118,8 +5567,7 @@ "psl": { "version": "1.1.29", "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", - "integrity": "sha1-YPWA02AXC7cip5fMcEQR5tqFDGc=", - "optional": true + "integrity": "sha1-YPWA02AXC7cip5fMcEQR5tqFDGc=" }, "pstree.remy": { "version": "1.1.0", @@ -5450,6 +5898,35 @@ } } }, + "request-promise": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz", + "integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=", + "requires": { + "bluebird": "^3.5.0", + "request-promise-core": "1.1.1", + "stealthy-require": "^1.1.0", + "tough-cookie": ">=2.3.3" + } + }, + "request-promise-core": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", + "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", + "requires": { + "lodash": "^4.13.1" + } + }, + "request-promise-native": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.5.tgz", + "integrity": "sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU=", + "requires": { + "request-promise-core": "1.1.1", + "stealthy-require": "^1.1.0", + "tough-cookie": ">=2.3.3" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5529,6 +6006,14 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, + "saxes": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.3.tgz", + "integrity": "sha512-Nc5DXc5A+m3rUDtkS+vHlBWKT7mCKjJPyia7f8YMW773hsXVv2wEHQZGE0zs4+5PLwz9U5Sbl/94Cnd9vHV7Bg==", + "requires": { + "xmlchars": "^1.3.1" + } + }, "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", @@ -5856,6 +6341,37 @@ "extend-shallow": "^3.0.0" } }, + "sshpk": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", + "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "dependencies": { + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, "stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -5888,6 +6404,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, "stream-combiner": { "version": "0.2.2", "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", @@ -5985,6 +6506,11 @@ "es6-symbol": "^3.1.1" } }, + "symbol-tree": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz", + "integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY=" + }, "term-size": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", @@ -6130,7 +6656,6 @@ "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", "integrity": "sha1-U/Nto/R3g7CSWvoG/587FlKA94E=", - "optional": true, "requires": { "psl": "^1.1.24", "punycode": "^1.4.1" @@ -6139,11 +6664,18 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", - "optional": true + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" } } }, + "tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "requires": { + "punycode": "^2.1.0" + } + }, "trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", @@ -6156,6 +6688,19 @@ "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=", "optional": true }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, "type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", @@ -6372,6 +6917,14 @@ "xdg-basedir": "^3.0.0" } }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", @@ -6434,6 +6987,23 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, "vinyl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz", @@ -6497,6 +7067,62 @@ "source-map": "^0.5.1" } }, + "w3c-hr-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz", + "integrity": "sha1-gqwr/2PZUOqeMYmlimViX+3xkEU=", + "requires": { + "browser-process-hrtime": "^0.1.2" + } + }, + "w3c-xmlserializer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.0.0.tgz", + "integrity": "sha512-0et1+9uXYiIRAecx1D5Z1nk60+vimniGdIKl4XjeqkWi6acoHNlXMv1VR5jV+jF4ooeO08oWbYxeAJOcon1oMA==", + "requires": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "requires": { + "iconv-lite": "0.4.24" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } + } + }, + "whatwg-mimetype": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.2.0.tgz", + "integrity": "sha512-5YSO1nMd5D1hY3WzAQV3PzZL83W3YeyR1yW9PcH26Weh1t+Vzh9B6XkDh7aXm83HBZ4nSMvkjvN2H2ySWIvBgw==" + }, + "whatwg-url": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", + "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -6554,6 +7180,11 @@ } } }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + }, "wrap-ansi": { "version": "2.1.0", "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", @@ -6580,12 +7211,30 @@ "signal-exit": "^3.0.2" } }, + "ws": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.0.tgz", + "integrity": "sha512-H3dGVdGvW2H8bnYpIDc3u3LH8Wue3Qh+Zto6aXXFzvESkTVT6rAfKR6tR/+coaUvxs8yHtmNV0uioBF62ZGSTg==", + "requires": { + "async-limiter": "~1.0.0" + } + }, "xdg-basedir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=", "dev": true }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "xmlchars": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-1.3.1.tgz", + "integrity": "sha512-tGkGJkN8XqCod7OT+EvGYK5Z4SfDQGD30zAa58OcnAa0RRWgzUEK72tkXhsX1FZd+rgnhRxFtmO+ihkp8LHSkw==" + }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index 3457652..2276b07 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,9 @@ "hacker-news-api": "^2.0.0", "http-status": "^1.2.0", "joi": "^14.0.4", - "pdf-parse": "^1.1.1" + "jsdom": "^13.0.0", + "pdf-parse": "^1.1.1", + "request-promise": "^4.2.2" }, "devDependencies": { "@babel/core": "^7.1.2", diff --git a/server/app/ContentIngester.js b/server/app/ContentIngester.js index fe7db5b..c700d88 100644 --- a/server/app/ContentIngester.js +++ b/server/app/ContentIngester.js @@ -1,7 +1,11 @@ import { MethodNotImplemented } from '../APIError'; class ContentIngester { - getContent(type) { + getContentLatest() { + throw MethodNotImplemented(); + } + + getContentAll() { throw MethodNotImplemented(); } } diff --git a/server/app/SEDTranscriptIngester.js b/server/app/SEDTranscriptIngester.js index 8c80778..c52f13b 100644 --- a/server/app/SEDTranscriptIngester.js +++ b/server/app/SEDTranscriptIngester.js @@ -1,47 +1,167 @@ +/* +- get posts from https://software-enginnering-daily-api.herokuapp.com/api/posts +- for each post + - get the url of the podcast episode + - get html of the link + - find the transcript url + - download and parse text from transcript + - upload to big table as unclassified +*/ + import config from '../../config/config'; import ContentIngester from './ContentIngester'; -import request from 'request'; -import fs from 'fs'; -import http from 'http'; +import request from 'request-promise'; import url from 'url'; import pdf from 'pdf-parse'; +import {JSDOM} from "jsdom"; + + +let TRANSCRIPT_REGEX = /sed([0-9\.]+)/; +let PDF_API_BASE_URL = "https://softwareengineeringdaily.com/wp-json/wp/v2/media?type=attachment&search=.pdf&"; +let PODCAST_CATEGORY = 14; +let POSTS_API_BASE_URL = `https://software-enginnering-daily-api.herokuapp.com/api/posts?categories=${PODCAST_CATEGORY}&`; + + +function getPosts(limit, page) { + let options = { + method: "GET", + uri: `${POSTS_API_BASE_URL}limit=${limit}&page=${page}` + } + return request(options).then(response => { + let allPosts = JSON.parse(response); + let posts = [] + for (var index in allPosts) { + posts.push({ + _id: allPosts[index]._id, + link: allPosts[index].link + }) + } + return posts; + }) +} + +function getPostTranscriptUrl(postUrl) { + let options = { + method: "GET", + uri: postUrl + } + return request(options).then(response => { + const dom = new JSDOM(response); + let list = dom.window.document.getElementsByClassName("post__content")[0].getElementsByTagName("p") + for (var i = 0; i < list.length; i++) { + if (list[i].innerHTML.indexOf("Transcript provided by We Edit Podcasts") > -1) { + console.log(list[i].innerHTML) + } + } + }) +} + function pdfToText(filename) { let dataBuffer = fs.readFileSync(filename); return pdf(dataBuffer) } +function getAllTranscripts(per_page, page) { + let options = { + method: "GET", + uri: `${PDF_API_BASE_URL}per_page=${per_page}&page=${page}` + } + return request(options).then(response => { + let allPdfs = JSON.parse(response); + let transcripts = [] + for (var index in allPdfs) { + if (allPdfs[index].slug.match(TRANSCRIPT_REGEX)) { + console.log(allPdfs[index]) + transcripts.push(allPdfs[index].guid.rendered) + } + } + return transcripts; + }) +} + +function downloadTranscript(file_url) { + return request({ + method: "GET", + uri: file_url, + encoding: null, + headers: { + "Content-type": "applcation/pdf" + } + }).then(response => { + return pdf(response) + }) +} + +function downloadAllTranscripts(allTranscriptUrls, index, callback) { + var index = index ? index : 0; + console.log('index:', index) + downloadTranscript(allTranscriptUrls[index]).then(result => { + console.log(result.info.Title) + // console.log(result.text) + // send result to big table with associated title and/or id + if (index < allTranscriptUrls.length) { + downloadAllTranscripts(allTranscriptUrls, index + 1, callback) + return null; + } else { + callback(null) + } + }).catch(err => { + callback(err) + }) +} + class SEDTranscriptIngester extends ContentIngester { constructor() { super(); } - getContent() { - var file_url = "https://softwareengineeringdaily.com/wp-content/uploads/2018/10/SED702-BloxRoute-Scaling-Bitcoin.pdf" - var DOWNLOAD_DIR = './downloads/'; - var options = { - host: url.parse(file_url).host, - port: 80, - path: url.parse(file_url).pathname - }; - - var file_name = url.parse(file_url).pathname.split('/').pop(); - var file = fs.createWriteStream(DOWNLOAD_DIR + file_name); - - return new Promise((resolve, reject) => { - http.get(options, function(res) { - res.on('data', function(data) { - file.write(data); - }).on('end', function() { - file.end(); - resolve() - }); - }); - }).then(() => { - return pdfToText(DOWNLOAD_DIR + file_name) - }).then(function(data) { - console.log(data.text); - }); + getContentAll() { + let allTranscriptUrls = [] + getAllTranscripts("100", "1").then(transcripts => { + return transcripts; + }) + .then(t => { + allTranscriptUrls = allTranscriptUrls.concat(t) + console.log(allTranscriptUrls.length) + return getAllTranscripts("100", "2") + }) + .then(t => { + allTranscriptUrls = allTranscriptUrls.concat(t) + console.log(allTranscriptUrls.length) + return getAllTranscripts("100", "3") + }) + .then(t => { + allTranscriptUrls = allTranscriptUrls.concat(t) + console.log(allTranscriptUrls.length) + return getAllTranscripts("100", "4") + }) + .then(t => { + allTranscriptUrls = allTranscriptUrls.concat(t) + console.log(allTranscriptUrls.length) + return getAllTranscripts("100", "5") + }) + .then(t => { + allTranscriptUrls = allTranscriptUrls.concat(t) + console.log(allTranscriptUrls.length) + downloadAllTranscripts(allTranscriptUrls, 0, () => { + console.log('done') + }) + }) + } + + getContentLatest() { + // getAllTranscripts("4", "1").then(transcripts => { + // downloadAllTranscripts(transcripts, 0, () => { + // console.log('done') + // }) + // }) + getPosts("1", "1").then(posts => { + posts.forEach(post => { + getPostTranscriptUrl(post.link) + }) + + }) } } diff --git a/server/app/big-table-schema.json b/server/app/big-table-schema.json new file mode 100644 index 0000000..9eb2e28 --- /dev/null +++ b/server/app/big-table-schema.json @@ -0,0 +1,18 @@ + +{ + "url": { + "internal": "https://www.softwaredaily.com/post/5be16aeae5e4bd0004b1c6e2", + "com": "https://softwareengineeringdaily.com/2018/11/06/oss-capital-with-joseph-jacks/" + }, + "title": "OSS Capital with Joseph Jacks", + "description": "", + "image_link": "https://i0.wp.com/softwareengineeringdaily.com/wp-content/uploads/2018/11/ossCapital.png?fit=800%2C800&ssl=1", + "ranking": { + "security": ".63", + "kubernetes": ".73", + "bitcoin" : ".22" + }, + "classified": "true" +} + +