using nodejs worker for better perf on largeeeeeee app

README.md

@@ -9,6 +9,7 @@ This package simplifies creating npm packages that work seamlessly across browse
 - **Universal Compatibility:** Works across browsers and Node.js.
 - **ESM and CJS Compilation:** Compiles code to both CJS and ESM formats.
 - **Simple and Lightweight:** Easy to use and maintains a small footprint.
+- **Multiple Threads** Can utilize multiple threads for extremely large projects.
 - **Path Conversion:** Converts TypeScript config paths to relative paths for compatibility.
 - **ESM `__dirname` and `__filename` Support:** Enables these variables for ESM compatibility.
 

package.json

@@ -1,13 +1,14 @@
 {
   "name": "npmize",
-  "version": "1.1.6",
+  "version": "1.1.7",
   "description": "Let's create an npm package without worrying about anything.",
   "bin": "./dist/index.js",
   "scripts": {
     "dev": "tsc -w",
     "build": "node ./build.cjs",
     "tsc": "tsc --noEmit --watch",
-    "lab": "run ./src/__lab__/index.ts"
+    "lab": "run ./src/__lab__/index.ts",
+    "lab-js": "node --watch ./dist/__lab__/index.js"
   },
   "dependencies": {
     "@babel/parser": "^7.25.6",

src/__lab__/index.ts

@@ -4,5 +4,6 @@ import { app } from '../app'
 import '../main'
 
 // app.start(['init', TEST_TARGET])
-app.start(['dev', TEST_TARGET, '--node'])
-// app.start(['build', TEST_TARGET])
+// app.start(['dev', TEST_TARGET, '--node'])
+app.start(['build', TEST_TARGET])
+// app.start(['build', TEST_TARGET, '--worker'])

src/app.ts

@@ -101,4 +101,11 @@ export const dev = app.create('dev', {
 export const build = app.create('build', {
   ...devAndBuild,
   description: 'Build the package for production',
+  flags: {
+    ...devAndBuild.flags,
+    worker: NoArg.boolean()
+      .aliases('w')
+      .default(false)
+      .description('Run the build process in a worker thread'),
+  },
 })

src/main.ts

@@ -72,7 +72,7 @@ app.build.on(([root = '.', railingArgs], options) => {
 
   build(rootPath, {
     ...options,
-    tsc: railingArgs,
+     tsc: railingArgs,
     tsConfig: { ...tsConfig, outDir: tsConfig.outDir },
   })
 })

src/program/build.ts

@@ -4,8 +4,9 @@ import { CompileOptions } from './types.t'
 import makeOutput from '../scripts/makeOutputFile'
 import { getNodeModulesTempDir } from '../utils'
 import { cleanDir, getAllFiles } from '../utils/fs'
+import ansiColors from 'ansi-colors'
 
-export default function (rootPath: string, options: CompileOptions) {
+export default async function (rootPath: string, options: BuildOptions) {
   console.log(`Build started at ${rootPath}`)
   console.log('')
 
@@ -15,18 +16,19 @@ export default function (rootPath: string, options: CompileOptions) {
     return runBuild(rootPath, options.tsConfig.outDir, options.module, options)
   }
 
-  runBuild(rootPath, options.tsConfig.outDir, 'cjs', options)
-  runBuild(rootPath, options.tsConfig.outDir, 'mjs', options)
+  await runBuild(rootPath, options.tsConfig.outDir, 'cjs', options)
+  await runBuild(rootPath, options.tsConfig.outDir, 'mjs', options)
 }
 
-function runBuild(
+async function runBuild(
   rootPath: string,
   fullOutDir: string,
-  moduleType: Exclude<CompileOptions['module'], undefined>,
-  options: Omit<CompileOptions, 'module' | 'outDir'>
+  moduleType: Exclude<BuildOptions['module'], undefined>,
+  options: Omit<BuildOptions, 'module' | 'outDir'>
 ) {
   console.log(`Building ${moduleType}...`)
 
+  console.time('Build time')
   const tempOutDir = getNodeModulesTempDir(rootPath, 'build-' + moduleType)
   cleanDir(tempOutDir)
 
@@ -38,11 +40,12 @@ function runBuild(
     moduleType === 'cjs' ? 'commonjs' : 'esnext',
   ])
 
-  const newFiles = getAllFiles(tempOutDir).map((file) => {
+  const promisesOfFiles = getAllFiles(tempOutDir).map((file) => {
     return makeOutput(file, {
       tempOutDir: tempOutDir,
       finalOutDir: fullOutDir,
       moduleType: moduleType,
+      useWorker: options.worker,
       pushNodeCode: options.node,
       tsConfig: {
         paths: options.tsConfig?.paths,
@@ -51,6 +54,9 @@ function runBuild(
     })
   })
 
+  const newFiles = await Promise.all(promisesOfFiles)
+  console.timeEnd('Build time')
+
   const fileSizes = newFiles
     .filter((a) => a.endsWith('js'))
     .map((file) => {
@@ -63,11 +69,17 @@ function runBuild(
 
   const totalSize = fileSizes.reduce((acc, { size }) => acc + size, 0)
   console.log(
-    `Built ${fileSizes.length} JavaScript files with a total size of ${(
-      totalSize / 1024
-    ).toFixed(2)}KB`
+    `Built ${ansiColors.yellow(
+      String(fileSizes.length)
+    )} JavaScript files with a total size of ${ansiColors.yellow(
+      (totalSize / 1024).toFixed(2)
+    )} KB`
   )
 
   cleanDir(tempOutDir, false)
   console.log('')
 }
+
+export type BuildOptions = CompileOptions & {
+  worker: boolean
+}

src/program/dev.ts

@@ -7,10 +7,7 @@ import { CompileOptions } from './types.t'
 import makeOutput from '../scripts/makeOutputFile'
 import { getNodeModulesTempDir } from '../utils'
 
-export default function (
-  rootPath: string,
-  options: CompileOptions & { focus: 'cjs' | 'mjs' }
-) {
+export default function (rootPath: string, options: DevOptions) {
   if (options.module) {
     return runDev(rootPath, options.tsConfig.outDir, options.module, options)
   }
@@ -22,16 +19,17 @@ export default function (
 function runDev(
   rootPath: string,
   shortOutDir: string,
-  moduleType: Exclude<CompileOptions['module'], undefined>,
-  options: Omit<CompileOptions, 'module' | 'outDir'> & { focus: 'cjs' | 'mjs' }
+  moduleType: Exclude<DevOptions['module'], undefined>,
+  options: Omit<DevOptions, 'module' | 'outDir'>
 ) {
   const tempOutDir = getNodeModulesTempDir(rootPath, 'dev-' + moduleType)
   const finalOutDir = path.resolve(shortOutDir)
 
   cleanDir(tempOutDir)
   cleanDir(finalOutDir)
   function updateFile(filePath: string) {
-    makeOutput(filePath, {
+    return makeOutput(filePath, {
+      useWorker: false,
       tempOutDir: tempOutDir,
       finalOutDir: finalOutDir,
       moduleType: moduleType,
@@ -65,3 +63,7 @@ function runDev(
     updateFile(filePath)
   })
 }
+
+export type DevOptions = CompileOptions & {
+  focus: 'cjs' | 'mjs'
+}

src/scripts/generateOutput.ts

@@ -0,0 +1,31 @@
+import path from 'path'
+import getNodeCode from './getNodeCode'
+import updateImports from '../updateImports'
+import { MakeOutputOptions } from './makeOutputFile'
+
+export default async function (
+  filePath: string,
+  fileContent: string,
+  options: MakeOutputOptions
+) {
+  const [tempFilePath, newFileContent] = await updateImports(
+    options.tempOutDir,
+    filePath,
+    fileContent,
+    options.moduleType,
+    options.tsConfig?.baseUrl,
+    options.tsConfig?.paths
+  )
+
+  const newFilePath = path.join(
+    options.finalOutDir,
+    path.relative(options.tempOutDir, tempFilePath)
+  )
+
+  const output =
+    options.pushNodeCode && options.moduleType === 'cjs'
+      ? getNodeCode(newFilePath, newFileContent) + newFileContent
+      : newFileContent
+
+  return [newFilePath, output] as const
+}

src/scripts/generateOutputWorker.ts

@@ -0,0 +1,17 @@
+import { parentPort } from 'worker_threads'
+import generateOutput from './generateOutput'
+import { MakeOutputOptions } from './makeOutputFile'
+
+parentPort!.on('message', (options: WorkerMessage) => {
+  generateOutput(options.filePath, options.fileContent, options.options).then(
+    (data) => {
+      parentPort!.postMessage(data)
+    }
+  )
+})
+
+export type WorkerMessage = {
+  filePath: string
+  fileContent: string
+  options: MakeOutputOptions
+}

src/scripts/makeOutputFile.ts

@@ -1,10 +1,11 @@
 import fs from 'fs'
 import path from 'path'
-import pushNodeCode from './getNodeCode'
-import updateImports from '../updateImports'
+import { Worker } from 'worker_threads'
 import { autoCreateDir } from '../utils/fs'
+import generateOutput from './generateOutput'
 
-type MakeOutputOptions = {
+export type MakeOutputOptions = {
+  useWorker: boolean
   tempOutDir: string
   finalOutDir: string
   moduleType: 'cjs' | 'mjs'
@@ -14,27 +15,38 @@ type MakeOutputOptions = {
     paths?: Record<string, string[]>
   }
 }
-export default function (filePath: string, options: MakeOutputOptions) {
-  const [tempFilePath, newFileContent] = updateImports(
-    options.tempOutDir,
-    filePath,
-    options.moduleType,
-    options.tsConfig?.baseUrl,
-    options.tsConfig?.paths
-  )
 
-  const newFilePath = path.join(
-    options.finalOutDir,
-    path.relative(options.tempOutDir, tempFilePath)
+export default async function (filePath: string, options: MakeOutputOptions) {
+  const fileContent = fs.readFileSync(filePath, 'utf-8')
+  const generator = options.useWorker ? generateOutputWorker : generateOutput
+  const [newFilePath, newFileContent] = await generator(
+    filePath,
+    fileContent,
+    options
   )
 
-  const contentWithNodeCode =
-    options.pushNodeCode && options.moduleType === 'cjs'
-      ? pushNodeCode(newFilePath, newFileContent) + newFileContent
-      : newFileContent
-
   autoCreateDir(path.dirname(newFilePath))
-  fs.writeFileSync(newFilePath, contentWithNodeCode)
+  fs.writeFileSync(newFilePath, newFileContent)
 
   return newFilePath
 }
+
+function generateOutputWorker(
+  filePath: string,
+  fileContent: string,
+  options: MakeOutputOptions
+): ReturnType<typeof generateOutput> {
+  const worker = new Worker(path.join(__dirname, 'generateOutputWorker.js'))
+  worker.postMessage({
+    filePath,
+    fileContent,
+    options,
+  })
+
+  return new Promise((resolve) => {
+    worker.on('message', (message) => {
+      resolve(message)
+      worker.terminate()
+    })
+  })
+}

src/updateImports/index.ts

@@ -1,51 +1,37 @@
-import fs from 'fs'
 import path from 'path'
-import * as node from './node'
 import * as utils from './utils'
 import { NodeType } from './types.t'
 import * as babel from '@babel/parser'
+import { Worker } from 'worker_threads'
 import { Statement } from '@babel/types'
+import { getImports, getRequires } from './node'
 import { resolveImportPath } from '../scripts/tsconfig'
 
-const getImports = (parsed: Statement[]) => {
-  return [
-    ...node.CallExpressionImport(parsed),
-    ...node.TSImportType(parsed),
-    ...node.ImportDeclaration_ExportNamedDeclaration_ExportAllDeclaration(
-      parsed
-    ),
-  ]
-}
-
-const getRequires = (parsed: Statement[]) => {
-  return node.CallExpressionRequire(parsed)
-}
-
-export default (
+export default async function (
   cwd: string,
   filePath: string,
+  compiledText: string,
   moduleType: 'cjs' | 'mjs',
   tsconfigBaseUrl: string | undefined,
   tsconfigPaths: Record<string, string[]> | undefined
-) => {
+) {
   const ext = '.' + moduleType
-  const dirPath = path.dirname(filePath)
-  const data = fs.readFileSync(filePath, 'utf-8')
+  const fileDirPath = path.dirname(filePath)
 
-  const parsedBody = babel.parse(data, {
+  const parsedBody = babel.parse(compiledText, {
     sourceType: 'module',
     plugins: ['typescript'],
     sourceFilename: filePath,
   }).program.body
 
-  const foundImportPaths =
-    filePath.endsWith('.ts') || moduleType === 'mjs'
-      ? getImports(parsedBody)
-      : getRequires(parsedBody)
+  const isModuleJs = filePath.endsWith('.ts') || moduleType === 'mjs'
+  const foundImportPaths = isModuleJs
+    ? getImports(parsedBody)
+    : getRequires(parsedBody)
 
   function updateRelativeImports(node: NodeType) {
     const shortPath = node.value.replace(/\.js$/i, '')
-    const fullPath = path.join(dirPath, node.value)
+    const fullPath = path.join(fileDirPath, node.value)
 
     const isExists = utils.isFileExists(fullPath)
     return isExists ? shortPath + ext : shortPath + '/index' + ext
@@ -74,7 +60,7 @@ export default (
   }
 
   const updatedData = utils.getUpdatedData(
-    data,
+    compiledText,
     foundImportPaths,
     (node: NodeType) => {
       if (node.value.startsWith('.')) {
@@ -95,3 +81,13 @@ export default (
   const newFilePath = utils.getNewFilePath(filePath, moduleType)
   return [newFilePath, updatedData] as const
 }
+
+function getRequiredStatements(type: 'import' | 'require', body: Statement[]) {
+  return new Promise<NodeType[]>((resolve) => {
+    const worker = new Worker(path.join(__dirname, './nodeWorker.js'))
+    worker.postMessage({ type, body })
+    worker.on('message', (message) => {
+      resolve(message)
+    })
+  })
+}