diff --git a/package.json b/package.json index 5e03f53..125c516 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "scripts": { "build": "zx ./scripts/build-wiki.mjs", - "download-installers": "rimraf ./files && zx ./scripts/download-installers.mjs", + "download-installers": "rimraf -g ./files/* && zx ./scripts/download-installers.mjs", "update": "npm-check-updates -u" }, "dependencies": { @@ -13,6 +13,7 @@ }, "devDependencies": { "html-minifier-terser": "7.2.0", + "node-fetch": "^3.3.2", "npm-check-updates": "^17.1.1", "rimraf": "^6.0.1", "shx": "0.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 342fee6..309a077 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,9 @@ importers: html-minifier-terser: specifier: 7.2.0 version: 7.2.0 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 npm-check-updates: specifier: ^17.1.1 version: 17.1.1 @@ -811,6 +814,7 @@ packages: resolution: {integrity: sha512-P8pqC6DFVLFRSxvGsV9ZCVaQXg5OIm11nAZt0r1bKANYKKpHHqd+MkoQGM2plGB0id2mC0BLYg65bLL9C2El0w==} engines: {node: '>=0.8.2'} hasBin: true + bundledDependencies: [] '@types/estree@0.0.39': resolution: {integrity: sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==} @@ -935,7 +939,7 @@ packages: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} bluebird@https://registry.npmmirror.com/bluebird/-/bluebird-3.7.2.tgz: - resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==, tarball: https://registry.npmmirror.com/bluebird/-/bluebird-3.7.2.tgz} + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} version: 3.7.2 boxen@4.2.0: @@ -1090,6 +1094,10 @@ packages: resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==} engines: {node: '>=8'} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + debug@4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -1224,6 +1232,10 @@ packages: fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -1246,6 +1258,10 @@ packages: resolution: {integrity: sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==} engines: {node: '>=14'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} engines: {node: '>=10'} @@ -1717,6 +1733,14 @@ packages: no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-releases@2.0.12: resolution: {integrity: sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==} @@ -2238,6 +2262,10 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@4.0.2: resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} @@ -3627,6 +3655,8 @@ snapshots: crypto-random-string@2.0.0: {} + data-uri-to-buffer@4.0.1: {} + debug@4.3.4: dependencies: ms: 2.1.2 @@ -3769,6 +3799,11 @@ snapshots: fast-json-stable-stringify@2.1.0: {} + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -3795,6 +3830,10 @@ snapshots: cross-spawn: 7.0.3 signal-exit: 4.1.0 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + fs-extra@9.1.0: dependencies: at-least-node: 1.0.0 @@ -4250,6 +4289,14 @@ snapshots: lower-case: 2.0.2 tslib: 2.5.2 + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-releases@2.0.12: {} node-releases@2.0.14: {} @@ -4784,6 +4831,8 @@ snapshots: dependencies: defaults: 1.0.4 + web-streams-polyfill@3.3.3: {} + webidl-conversions@4.0.2: {} whatwg-url@7.1.0: diff --git a/scripts/download-installers.mjs b/scripts/download-installers.mjs index 9d1a6bc..e99c975 100644 --- a/scripts/download-installers.mjs +++ b/scripts/download-installers.mjs @@ -6,6 +6,7 @@ import { Readable } from 'stream'; import { finished } from 'stream/promises'; import { backOff } from 'exponential-backoff'; import { HttpsProxyAgent } from 'https-proxy-agent'; +import fetch from 'node-fetch'; require('dotenv').config(); const { GITHUB_TOKEN } = process.env; @@ -25,6 +26,10 @@ const latestDesktopReleaseData = await fetch('https://api.github.com/repos/tiddl const latestMobileReleaseData = await fetch('https://api.github.com/repos/tiddly-gittly/TidGi-Mobile/releases/latest').then( async (response) => await response.json(), ); +if (typeof latestDesktopReleaseData.tag_name === 'undefined') { + console.warn(latestDesktopReleaseData); + throw new Error('Try add github token to .env file'); +} const latestDesktopVersion = latestDesktopReleaseData.tag_name.replace('v', ''); const latestMobileVersion = latestMobileReleaseData.tag_name.replace('v', ''); const desktopUrls = latestDesktopReleaseData.assets.map((asset) => asset.browser_download_url); @@ -36,9 +41,6 @@ console.log(mobileUrls); async function downloadAsset(asset, rename) { const fileName = rename(asset.name); console.log(`Downloading ${fileName} from ${asset.browser_download_url}`); - try { - await unlink(destination); - } catch {} const headers = { Accept: 'application/octet-stream', 'User-Agent': '@terascope/fetch-github-release', @@ -47,11 +49,25 @@ async function downloadAsset(asset, rename) { if (GITHUB_TOKEN) { headers.Authorization = `token ${GITHUB_TOKEN}`; } - const { body } = await fetch(asset.url, { headers, agent }); const destination = path.join(__dirname, `../files/${fileName}`); - const fileStream = fs.createWriteStream(destination, { flags: 'wx' }); - await finished(body.pipe(fileStream)); - console.log(`Done ${fileName}`); + try { + await unlink(destination); + } catch (error) { + if (error.code !== 'ENOENT') console.log(`File ${fileName}cannot be deleted`, error); + } + try { + const response = await fetch(asset.url, { headers }); + if (!response.ok) { + throw new Error(`Failed to fetch ${asset.url}: ${response.statusText}`); + } + // try uncheck "readonly" on folder properties, if encounter "EPERM" error. + const fileStream = fs.createWriteStream(destination, { flags: 'wx' }); + await finished(response.body.pipe(fileStream)); + console.log(`Done ${fileName}`); + } catch (error) { + console.log(`Error downloading ${fileName}`, error); + throw error; + } } let chunkCounter = 0; @@ -76,12 +92,18 @@ await Promise.all([ { numOfAttempts: 10000, jitter: 'full' }, ); }), - ...latestMobileReleaseData.assets.map((asset) => - backOff(() => - downloadAsset(asset, (name) => { - const fileName = name.replace('app-release-signed', 'TidGi-Mobile'); - return fileName; - }), - ), - ), + ...latestMobileReleaseData.assets.map(async (asset) => { + await Promise.delay(5000 * Math.random()); + backOff(async () => { + console.log(`backoff retry ${asset.name}`); + await downloadAsset( + asset, + (name) => { + const fileName = name.replace('app-release-signed', 'TidGi-Mobile'); + return fileName; + }, + { numOfAttempts: 10000, jitter: 'full' }, + ); + }); + }), ]);