From 5ef44ef7673b51c49df911f87651e8e0185ce438 Mon Sep 17 00:00:00 2001 From: Nazmus Sayad <87106526+NazmusSayad@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:44:42 +0600 Subject: [PATCH] using nodejs worker for better perf on largeeeeeee app --- README.md | 1 + package-lock.json | 4 +-- package.json | 5 +-- src/__lab__/index.ts | 5 +-- src/app.ts | 7 ++++ src/main.ts | 2 +- src/program/build.ts | 32 ++++++++++++------ src/program/dev.ts | 16 +++++---- src/scripts/generateOutput.ts | 31 +++++++++++++++++ src/scripts/generateOutputWorker.ts | 17 ++++++++++ src/scripts/makeOutputFile.ts | 52 ++++++++++++++++++----------- src/updateImports/index.ts | 50 +++++++++++++-------------- src/updateImports/node.ts | 20 ++++++++--- 13 files changed, 167 insertions(+), 75 deletions(-) create mode 100644 src/scripts/generateOutput.ts create mode 100644 src/scripts/generateOutputWorker.ts diff --git a/README.md b/README.md index bc99bef..4642c2d 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This package simplifies creating npm packages that work seamlessly across browse - **Universal Compatibility:** Works across browsers and Node.js. - **ESM and CJS Compilation:** Compiles code to both CJS and ESM formats. - **Simple and Lightweight:** Easy to use and maintains a small footprint. +- **Multiple Threads** Can utilize multiple threads for extremely large projects. - **Path Conversion:** Converts TypeScript config paths to relative paths for compatibility. - **ESM `__dirname` and `__filename` Support:** Enables these variables for ESM compatibility. diff --git a/package-lock.json b/package-lock.json index b285507..c45b715 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "npmize", - "version": "1.1.5", + "version": "1.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "npmize", - "version": "1.1.5", + "version": "1.1.6", "license": "ISC", "dependencies": { "@babel/parser": "^7.25.6", diff --git a/package.json b/package.json index fa9980f..9a9e173 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,14 @@ { "name": "npmize", - "version": "1.1.6", + "version": "1.1.7", "description": "Let's create an npm package without worrying about anything.", "bin": "./dist/index.js", "scripts": { "dev": "tsc -w", "build": "node ./build.cjs", "tsc": "tsc --noEmit --watch", - "lab": "run ./src/__lab__/index.ts" + "lab": "run ./src/__lab__/index.ts", + "lab-js": "node --watch ./dist/__lab__/index.js" }, "dependencies": { "@babel/parser": "^7.25.6", diff --git a/src/__lab__/index.ts b/src/__lab__/index.ts index 26382ed..c1c5c2e 100644 --- a/src/__lab__/index.ts +++ b/src/__lab__/index.ts @@ -4,5 +4,6 @@ import { app } from '../app' import '../main' // app.start(['init', TEST_TARGET]) -app.start(['dev', TEST_TARGET, '--node']) -// app.start(['build', TEST_TARGET]) +// app.start(['dev', TEST_TARGET, '--node']) +app.start(['build', TEST_TARGET]) +// app.start(['build', TEST_TARGET, '--worker']) diff --git a/src/app.ts b/src/app.ts index 7493944..bdde283 100644 --- a/src/app.ts +++ b/src/app.ts @@ -101,4 +101,11 @@ export const dev = app.create('dev', { export const build = app.create('build', { ...devAndBuild, description: 'Build the package for production', + flags: { + ...devAndBuild.flags, + worker: NoArg.boolean() + .aliases('w') + .default(false) + .description('Run the build process in a worker thread'), + }, }) diff --git a/src/main.ts b/src/main.ts index 3ff1232..626c39a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -72,7 +72,7 @@ app.build.on(([root = '.', railingArgs], options) => { build(rootPath, { ...options, - tsc: railingArgs, + tsc: railingArgs, tsConfig: { ...tsConfig, outDir: tsConfig.outDir }, }) }) diff --git a/src/program/build.ts b/src/program/build.ts index a51e03f..856b1ef 100644 --- a/src/program/build.ts +++ b/src/program/build.ts @@ -4,8 +4,9 @@ import { CompileOptions } from './types.t' import makeOutput from '../scripts/makeOutputFile' import { getNodeModulesTempDir } from '../utils' import { cleanDir, getAllFiles } from '../utils/fs' +import ansiColors from 'ansi-colors' -export default function (rootPath: string, options: CompileOptions) { +export default async function (rootPath: string, options: BuildOptions) { console.log(`Build started at ${rootPath}`) console.log('') @@ -15,18 +16,19 @@ export default function (rootPath: string, options: CompileOptions) { return runBuild(rootPath, options.tsConfig.outDir, options.module, options) } - runBuild(rootPath, options.tsConfig.outDir, 'cjs', options) - runBuild(rootPath, options.tsConfig.outDir, 'mjs', options) + await runBuild(rootPath, options.tsConfig.outDir, 'cjs', options) + await runBuild(rootPath, options.tsConfig.outDir, 'mjs', options) } -function runBuild( +async function runBuild( rootPath: string, fullOutDir: string, - moduleType: Exclude, - options: Omit + moduleType: Exclude, + options: Omit ) { console.log(`Building ${moduleType}...`) + console.time('Build time') const tempOutDir = getNodeModulesTempDir(rootPath, 'build-' + moduleType) cleanDir(tempOutDir) @@ -38,11 +40,12 @@ function runBuild( moduleType === 'cjs' ? 'commonjs' : 'esnext', ]) - const newFiles = getAllFiles(tempOutDir).map((file) => { + const promisesOfFiles = getAllFiles(tempOutDir).map((file) => { return makeOutput(file, { tempOutDir: tempOutDir, finalOutDir: fullOutDir, moduleType: moduleType, + useWorker: options.worker, pushNodeCode: options.node, tsConfig: { paths: options.tsConfig?.paths, @@ -51,6 +54,9 @@ function runBuild( }) }) + const newFiles = await Promise.all(promisesOfFiles) + console.timeEnd('Build time') + const fileSizes = newFiles .filter((a) => a.endsWith('js')) .map((file) => { @@ -63,11 +69,17 @@ function runBuild( const totalSize = fileSizes.reduce((acc, { size }) => acc + size, 0) console.log( - `Built ${fileSizes.length} JavaScript files with a total size of ${( - totalSize / 1024 - ).toFixed(2)}KB` + `Built ${ansiColors.yellow( + String(fileSizes.length) + )} JavaScript files with a total size of ${ansiColors.yellow( + (totalSize / 1024).toFixed(2) + )} KB` ) cleanDir(tempOutDir, false) console.log('') } + +export type BuildOptions = CompileOptions & { + worker: boolean +} diff --git a/src/program/dev.ts b/src/program/dev.ts index be660d5..7649f12 100644 --- a/src/program/dev.ts +++ b/src/program/dev.ts @@ -7,10 +7,7 @@ import { CompileOptions } from './types.t' import makeOutput from '../scripts/makeOutputFile' import { getNodeModulesTempDir } from '../utils' -export default function ( - rootPath: string, - options: CompileOptions & { focus: 'cjs' | 'mjs' } -) { +export default function (rootPath: string, options: DevOptions) { if (options.module) { return runDev(rootPath, options.tsConfig.outDir, options.module, options) } @@ -22,8 +19,8 @@ export default function ( function runDev( rootPath: string, shortOutDir: string, - moduleType: Exclude, - options: Omit & { focus: 'cjs' | 'mjs' } + moduleType: Exclude, + options: Omit ) { const tempOutDir = getNodeModulesTempDir(rootPath, 'dev-' + moduleType) const finalOutDir = path.resolve(shortOutDir) @@ -31,7 +28,8 @@ function runDev( cleanDir(tempOutDir) cleanDir(finalOutDir) function updateFile(filePath: string) { - makeOutput(filePath, { + return makeOutput(filePath, { + useWorker: false, tempOutDir: tempOutDir, finalOutDir: finalOutDir, moduleType: moduleType, @@ -65,3 +63,7 @@ function runDev( updateFile(filePath) }) } + +export type DevOptions = CompileOptions & { + focus: 'cjs' | 'mjs' +} diff --git a/src/scripts/generateOutput.ts b/src/scripts/generateOutput.ts new file mode 100644 index 0000000..90d720a --- /dev/null +++ b/src/scripts/generateOutput.ts @@ -0,0 +1,31 @@ +import path from 'path' +import getNodeCode from './getNodeCode' +import updateImports from '../updateImports' +import { MakeOutputOptions } from './makeOutputFile' + +export default async function ( + filePath: string, + fileContent: string, + options: MakeOutputOptions +) { + const [tempFilePath, newFileContent] = await updateImports( + options.tempOutDir, + filePath, + fileContent, + options.moduleType, + options.tsConfig?.baseUrl, + options.tsConfig?.paths + ) + + const newFilePath = path.join( + options.finalOutDir, + path.relative(options.tempOutDir, tempFilePath) + ) + + const output = + options.pushNodeCode && options.moduleType === 'cjs' + ? getNodeCode(newFilePath, newFileContent) + newFileContent + : newFileContent + + return [newFilePath, output] as const +} diff --git a/src/scripts/generateOutputWorker.ts b/src/scripts/generateOutputWorker.ts new file mode 100644 index 0000000..9ad1898 --- /dev/null +++ b/src/scripts/generateOutputWorker.ts @@ -0,0 +1,17 @@ +import { parentPort } from 'worker_threads' +import generateOutput from './generateOutput' +import { MakeOutputOptions } from './makeOutputFile' + +parentPort!.on('message', (options: WorkerMessage) => { + generateOutput(options.filePath, options.fileContent, options.options).then( + (data) => { + parentPort!.postMessage(data) + } + ) +}) + +export type WorkerMessage = { + filePath: string + fileContent: string + options: MakeOutputOptions +} diff --git a/src/scripts/makeOutputFile.ts b/src/scripts/makeOutputFile.ts index 521f5de..3830a18 100644 --- a/src/scripts/makeOutputFile.ts +++ b/src/scripts/makeOutputFile.ts @@ -1,10 +1,11 @@ import fs from 'fs' import path from 'path' -import pushNodeCode from './getNodeCode' -import updateImports from '../updateImports' +import { Worker } from 'worker_threads' import { autoCreateDir } from '../utils/fs' +import generateOutput from './generateOutput' -type MakeOutputOptions = { +export type MakeOutputOptions = { + useWorker: boolean tempOutDir: string finalOutDir: string moduleType: 'cjs' | 'mjs' @@ -14,27 +15,38 @@ type MakeOutputOptions = { paths?: Record } } -export default function (filePath: string, options: MakeOutputOptions) { - const [tempFilePath, newFileContent] = updateImports( - options.tempOutDir, - filePath, - options.moduleType, - options.tsConfig?.baseUrl, - options.tsConfig?.paths - ) - const newFilePath = path.join( - options.finalOutDir, - path.relative(options.tempOutDir, tempFilePath) +export default async function (filePath: string, options: MakeOutputOptions) { + const fileContent = fs.readFileSync(filePath, 'utf-8') + const generator = options.useWorker ? generateOutputWorker : generateOutput + const [newFilePath, newFileContent] = await generator( + filePath, + fileContent, + options ) - const contentWithNodeCode = - options.pushNodeCode && options.moduleType === 'cjs' - ? pushNodeCode(newFilePath, newFileContent) + newFileContent - : newFileContent - autoCreateDir(path.dirname(newFilePath)) - fs.writeFileSync(newFilePath, contentWithNodeCode) + fs.writeFileSync(newFilePath, newFileContent) return newFilePath } + +function generateOutputWorker( + filePath: string, + fileContent: string, + options: MakeOutputOptions +): ReturnType { + const worker = new Worker(path.join(__dirname, 'generateOutputWorker.js')) + worker.postMessage({ + filePath, + fileContent, + options, + }) + + return new Promise((resolve) => { + worker.on('message', (message) => { + resolve(message) + worker.terminate() + }) + }) +} diff --git a/src/updateImports/index.ts b/src/updateImports/index.ts index a82713a..23d85e0 100644 --- a/src/updateImports/index.ts +++ b/src/updateImports/index.ts @@ -1,51 +1,37 @@ -import fs from 'fs' import path from 'path' -import * as node from './node' import * as utils from './utils' import { NodeType } from './types.t' import * as babel from '@babel/parser' +import { Worker } from 'worker_threads' import { Statement } from '@babel/types' +import { getImports, getRequires } from './node' import { resolveImportPath } from '../scripts/tsconfig' -const getImports = (parsed: Statement[]) => { - return [ - ...node.CallExpressionImport(parsed), - ...node.TSImportType(parsed), - ...node.ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration( - parsed - ), - ] -} - -const getRequires = (parsed: Statement[]) => { - return node.CallExpressionRequire(parsed) -} - -export default ( +export default async function ( cwd: string, filePath: string, + compiledText: string, moduleType: 'cjs' | 'mjs', tsconfigBaseUrl: string | undefined, tsconfigPaths: Record | undefined -) => { +) { const ext = '.' + moduleType - const dirPath = path.dirname(filePath) - const data = fs.readFileSync(filePath, 'utf-8') + const fileDirPath = path.dirname(filePath) - const parsedBody = babel.parse(data, { + const parsedBody = babel.parse(compiledText, { sourceType: 'module', plugins: ['typescript'], sourceFilename: filePath, }).program.body - const foundImportPaths = - filePath.endsWith('.ts') || moduleType === 'mjs' - ? getImports(parsedBody) - : getRequires(parsedBody) + const isModuleJs = filePath.endsWith('.ts') || moduleType === 'mjs' + const foundImportPaths = isModuleJs + ? getImports(parsedBody) + : getRequires(parsedBody) function updateRelativeImports(node: NodeType) { const shortPath = node.value.replace(/\.js$/i, '') - const fullPath = path.join(dirPath, node.value) + const fullPath = path.join(fileDirPath, node.value) const isExists = utils.isFileExists(fullPath) return isExists ? shortPath + ext : shortPath + '/index' + ext @@ -74,7 +60,7 @@ export default ( } const updatedData = utils.getUpdatedData( - data, + compiledText, foundImportPaths, (node: NodeType) => { if (node.value.startsWith('.')) { @@ -95,3 +81,13 @@ export default ( const newFilePath = utils.getNewFilePath(filePath, moduleType) return [newFilePath, updatedData] as const } + +function getRequiredStatements(type: 'import' | 'require', body: Statement[]) { + return new Promise((resolve) => { + const worker = new Worker(path.join(__dirname, './nodeWorker.js')) + worker.postMessage({ type, body }) + worker.on('message', (message) => { + resolve(message) + }) + }) +} diff --git a/src/updateImports/node.ts b/src/updateImports/node.ts index b0ac89a..a8fb208 100644 --- a/src/updateImports/node.ts +++ b/src/updateImports/node.ts @@ -1,13 +1,13 @@ import { Statement } from '@babel/types' import { findNestedItems, isOkString, parseString } from './utils' -export const TSImportType = (parsed: Statement[]) => { +const TSImportType = (parsed: Statement[]) => { return findNestedItems(parsed, 'type', 'TSImportType') .filter((node) => isOkString(node.argument)) .map((node) => parseString(node.argument)) } -export const ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration = ( +const ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration = ( parsed: Statement[] ) => { return [ @@ -21,7 +21,7 @@ export const ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration = ( .map((node) => parseString(node.source)) } -export const CallExpressionImport = (parsed: Statement[]) => { +const CallExpressionImport = (parsed: Statement[]) => { return findNestedItems(parsed, 'type', 'CallExpression') .filter( (node) => @@ -33,7 +33,7 @@ export const CallExpressionImport = (parsed: Statement[]) => { .map((node) => parseString(node.arguments[0])) } -export const CallExpressionRequire = (parsed: Statement[]) => { +const CallExpressionRequire = (parsed: Statement[]) => { return findNestedItems(parsed, 'type', 'CallExpression') .filter( (node) => @@ -45,3 +45,15 @@ export const CallExpressionRequire = (parsed: Statement[]) => { ) .map((node) => parseString(node.arguments[0])) } + +export function getImports(parsed: Statement[]) { + return [ + ...CallExpressionImport(parsed), + ...TSImportType(parsed), + ...ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration(parsed), + ] +} + +export function getRequires(parsed: Statement[]) { + return CallExpressionRequire(parsed) +}