From a86308b69f340fb16607d7af74c27ab81d14322d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 4 Jan 2025 04:01:47 +0800 Subject: [PATCH] Perf: run a few task during downloading previous dist --- Build/build-cdn-download-conf.ts | 11 ++++-- Build/build-common.ts | 24 ++++++------ Build/build-reject-domainset.ts | 4 +- Build/build-sspanel-appprofile.ts | 4 +- Build/download-mock-assets.ts | 50 +++++++++++++++--------- Build/download-previous-build.ts | 7 ++-- Build/index.ts | 63 ++++++++++--------------------- Build/lib/get-phishing-domains.ts | 2 - 8 files changed, 81 insertions(+), 84 deletions(-) diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index 5853984d2..7f407f4e8 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -48,18 +48,21 @@ const getS3OSSDomainsPromise = (async (): Promise> => { return S3OSSDomains; })(); +const cdnDomainsListPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf')); +const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf')); +const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')); + export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => { const [ S3OSSDomains, - cdnDomainsList, downloadDomainSet, steamDomainSet ] = await Promise.all([ getS3OSSDomainsPromise, - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf')), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf')), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')) + cdnDomainsListPromise, + downloadDomainSetPromise, + steamDomainSetPromise ]); // Move S3 domains to download domain set, since S3 files may be large diff --git a/Build/build-common.ts b/Build/build-common.ts index 8e04e61e3..1a16dc9b7 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -18,20 +18,22 @@ const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames '; const domainsetSrcFolder = 'domainset' + path.sep; -export const buildCommon = task(require.main === module, __filename)(async (span) => { - const promises: Array> = []; +const clawSourceDirPromise = new Fdir() + .withRelativePaths() + .filter((filepath, isDirectory) => { + if (isDirectory) return true; + + const extname = path.extname(filepath); - const paths = await new Fdir() - .withRelativePaths() - .filter((filepath, isDirectory) => { - if (isDirectory) return true; + return !(extname === '.js' || extname === '.ts'); + }) + .crawl(SOURCE_DIR) + .withPromise(); - const extname = path.extname(filepath); +export const buildCommon = task(require.main === module, __filename)(async (span) => { + const promises: Array> = []; - return !(extname === '.js' || extname === '.ts'); - }) - .crawl(SOURCE_DIR) - .withPromise(); + const paths = await clawSourceDirPromise; for (let i = 0, len = paths.length; i < len; i++) { const relativePath = paths[i]; diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 35b7f910b..39999ee07 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -19,6 +19,8 @@ import { appendArrayInPlace } from './lib/append-array-in-place'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { DomainsetOutput } from './lib/create-file'; +const readLocalNonIpRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')); + export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { const rejectBaseDescription = [ ...SHARED_DESCRIPTION, @@ -98,7 +100,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as addArrayElementsToSet(filterRuleWhitelistDomainSets, black); })), getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')).then(appendArrayToRejectOutput), + readLocalNonIpRejectRulesetPromise.then(appendArrayToRejectOutput), // Dedupe domainSets // span.traceChildAsync('collect black keywords/suffixes', async () => /** diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index a6063e03e..57ae22064 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -23,6 +23,8 @@ const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: bo ['Final Match', true, true] ]; +const steamDomainsPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')); + /** * This only generates a simplified version, for under-used users only. */ @@ -56,7 +58,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam // stream - domains AllStreamServices.flatMap((i) => i.rules), // steam - domains - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')), + steamDomainsPromise, // global - domains readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/global.conf')), readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')), diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index fe0357b06..edcecca11 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -14,20 +14,36 @@ const ASSETS_LIST = { 'amazon-adsystem-com_amazon-apstag.js': 'https://unpkg.com/@adguard/scriptlets@latest/dist/redirect-files/amazon-apstag.js' } as const; -export const downloadMockAssets = task(require.main === module, __filename)((span) => Promise.all(Object.entries(ASSETS_LIST).map( - ([filename, url]) => span - .traceChildAsync(url, async () => { - const res = await $fetch(url); - if (!res.body) { - throw new Error(`Empty body from ${url}`); - } - - await mkdirp(OUTPUT_MOCK_DIR); - const src = path.join(OUTPUT_MOCK_DIR, filename); - - return pipeline( - res.body, - fs.createWriteStream(src, 'utf-8') - ); - }) -))); +export const downloadMockAssets = task(require.main === module, __filename)(async (span) => { + const p = mkdirp(OUTPUT_MOCK_DIR); + if (p) { + await p; + } + + return Promise.all(Object.entries(ASSETS_LIST).map( + ([filename, url]) => span + .traceChildAsync(url, async () => { + const res = await $fetch(url); + if (!res.ok) { + console.error(`Failed to download ${url}`); + + // we can safely skip this since we can always use previous version + return; + } + + if (!res.body) { + console.error(`Empty body from ${url}`); + + // we can safely skip this since we can always use previous version + return; + } + + const src = path.join(OUTPUT_MOCK_DIR, filename); + + return pipeline( + res.body, + fs.createWriteStream(src, 'utf-8') + ); + }) + )); +}); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 005c45b26..584d7545e 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -7,14 +7,13 @@ import type { Headers as TarEntryHeaders } from 'tar-fs'; import zlib from 'node:zlib'; import undici from 'undici'; import picocolors from 'picocolors'; +import { PUBLIC_DIR } from './constants/dir'; const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master'; const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz'; export const downloadPreviousBuild = task(require.main === module, __filename)(async (span) => { - const publicDir = path.resolve(__dirname, '..', 'public'); - - if (fs.existsSync(publicDir)) { + if (fs.existsSync(PUBLIC_DIR)) { console.log(picocolors.blue('Public directory exists, skip downloading previous build')); return; } @@ -61,7 +60,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a const gunzip = zlib.createGunzip(); const extract = tarExtract( - publicDir, + PUBLIC_DIR, { ignore(_: string, header?: TarEntryHeaders) { if (header) { diff --git a/Build/index.ts b/Build/index.ts index acdba0d45..9b16fba2e 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -62,55 +62,30 @@ process.on('unhandledRejection', (reason) => { try { const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan); - const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan)); - const buildRejectIPListPromise = downloadPreviousBuildPromise.then(() => buildRejectIPList(rootSpan)); - const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan)); - const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnDownloadConf(rootSpan)); - const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan)); - const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan)); - const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan)); - const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan)); - - const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR(rootSpan); - - // const buildInternalChnDomainsPromise = buildInternalChnDomains(); - const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan)); - - const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan)); - const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan)); - - const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan)); - - const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan)); - - const buildSSPanelUIMAppProfilePromise = downloadPreviousBuildPromise.then(() => buildSSPanelUIMAppProfile(rootSpan)); - - const downloadMockAssetsPromise = downloadMockAssets(rootSpan); - - const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan)); await Promise.all([ downloadPreviousBuildPromise, buildCommonPromise, - buildRejectIPListPromise, - buildAppleCdnPromise, - buildCdnConfPromise, - buildRejectDomainSetPromise, - buildTelegramCIDRPromise, - buildChnCidrPromise, - buildSpeedtestDomainSetPromise, - buildInternalReverseChnCIDRPromise, - buildInternalReverseChnCIDRPromise, - // buildInternalChnDomainsPromise, - buildDomesticRulesetPromise, - buildRedirectModulePromise, - buildAlwaysRealIPModulePromise, - buildStreamServicePromise, - buildMicrosoftCdnPromise, - buildSSPanelUIMAppProfilePromise, - buildCloudMounterRulesPromise, - downloadMockAssetsPromise + downloadPreviousBuildPromise.then(() => buildRejectIPList(rootSpan)), + downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan)), + downloadPreviousBuildPromise.then(() => buildCdnDownloadConf(rootSpan)), + downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan)), + downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan)), + downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan)), + downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan)), + buildInternalReverseChnCIDR(rootSpan), + downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan)), + downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan)), + downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan)), + downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan)), + downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan)), + Promise.all([ + downloadPreviousBuildPromise, + buildCommonPromise + ]).then(() => buildSSPanelUIMAppProfile(rootSpan)), + downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan)), + downloadMockAssets(rootSpan) ]); await Promise.all([ diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 5b44ccd57..46784ae8d 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -219,8 +219,6 @@ export function getPhishingDomains(parentSpan: Span) { return domainArr; }); - console.log({ len: domainArr.length }); - return span.traceChildAsync( 'process phishing domain set', () => processPhihsingDomains(domainArr)