diff --git a/.github/actions/pnpm/action.yml b/.github/actions/pnpm/action.yml index 734030f2b00..682d1337f90 100644 --- a/.github/actions/pnpm/action.yml +++ b/.github/actions/pnpm/action.yml @@ -17,7 +17,7 @@ runs: - name: Setup node uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: - node-version: '18' + node-version: '20' cache: 'pnpm' - name: Install dependencies diff --git a/.github/actions/pretest/action.yml b/.github/actions/pretest/action.yml index b1828cdf8e7..af33d8b535c 100644 --- a/.github/actions/pretest/action.yml +++ b/.github/actions/pretest/action.yml @@ -28,7 +28,7 @@ runs: - name: Setup node uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: - node-version: '18' + node-version: '20' cache: 'pnpm' - name: Install dependencies diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 27ee1c47856..bee78facd7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: testFiles: - - ArtifactPublisherTest,BuildTest,ExtraBuildTest,RepoSlugTest,binDownloadTest,configurationValidationTest,filenameUtilTest,filesTest,globTest,ignoreTest,macroExpanderTest,mainEntryTest,urlUtilTest,extraMetadataTest,linuxArchiveTest,linuxPackagerTest,HoistedNodeModuleTest,MemoLazyTest + - ArtifactPublisherTest,BuildTest,ExtraBuildTest,RepoSlugTest,binDownloadTest,configurationValidationTest,filenameUtilTest,filesTest,globTest,ignoreTest,macroExpanderTest,mainEntryTest,urlUtilTest,extraMetadataTest,linuxArchiveTest,linuxPackagerTest,HoistedNodeModuleTest,MemoLazyTest,HoistTest - snapTest,debTest,fpmTest,protonTest steps: - name: Checkout code repository diff --git a/README.md b/README.md index 81304f18560..b4c1eee4f63 100644 --- a/README.md +++ b/README.md @@ -100,21 +100,6 @@ See the full documentation on [electron.build](https://www.electron.build). `yarn add electron-builder --dev` -### Note for PNPM - -In order to use with `pnpm`, you'll need to adjust your `.npmrc` to use any one the following approaches in order for your dependencies to be bundled correctly (ref: [#6389](https://github.com/electron-userland/electron-builder/issues/6289#issuecomment-1042620422)): -``` -node-linker=hoisted -``` -``` -public-hoist-pattern=* -``` -``` -shamefully-hoist=true -``` - -Note: Setting shamefully-hoist to true is the same as setting public-hoist-pattern to *. - ### Note for Yarn 3 Yarn 3 use PnP by default, but electron-builder still need node-modules(ref: [yarnpkg/berry#4804](https://github.com/yarnpkg/berry/issues/4804#issuecomment-1234407305)). Add configuration in the `.yarnrc.yaml` as follows: diff --git a/packages/app-builder-lib/src/node-module-collector/hoist.ts b/packages/app-builder-lib/src/node-module-collector/hoist.ts new file mode 100644 index 00000000000..0946f29bb7f --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/hoist.ts @@ -0,0 +1,1110 @@ +// copy from https://github.com/yarnpkg/berry/blob/master/packages/yarnpkg-nm/sources/hoist.ts +// commit: d63d411bcc5adcbffd198b8987c5a14c81eaf669 +// fix(nm): optimize hoisting by treating peer deps same as other deps (#6517) +/** + * High-level node_modules hoisting algorithm recipe + * + * 1. Take input dependency graph and start traversing it, + * as you visit new node in the graph - clone it if there can be multiple paths + * to access the node from the graph root to the node, e.g. essentially represent + * the graph with a tree as you go, to make hoisting possible. + * 2. You want to hoist every node possible to the top root node first, + * then to each of its children etc, so you need to keep track what is your current + * root node into which you are hoisting + * 3. Traverse the dependency graph from the current root node and for each package name + * that can be potentially hoisted to the current root node build a list of idents + * in descending hoisting preference. You will check in next steps whether most preferred ident + * for the given package name can be hoisted first, and if not, then you check the + * less preferred ident, etc, until either some ident will be hoisted + * or you run out of idents to check + * (no need to convert the graph to the tree when you build this preference map). + * 4. The children of the root node are already "hoisted", so you need to start + * from the dependencies of these children. You take some child and + * sort its dependencies so that regular dependencies without peer dependencies + * will come first and then those dependencies that peer depend on them. + * This is needed to make algorithm more efficient and hoist nodes which are easier + * to hoist first and then handle peer dependent nodes. + * 5. You take this sorted list of dependencies and check if each of them can be + * hoisted to the current root node. To answer is the node can be hoisted you check + * your constraints - require promise and peer dependency promise. + * The possible answers can be: YES - the node is hoistable to the current root, + * NO - the node is not hoistable to the current root + * and DEPENDS - the node is hoistable to the root if nodes X, Y, Z are hoistable + * to the root. The case DEPENDS happens when all the require and other + * constraints are met, except peer dependency constraints. Note, that the nodes + * that are not package idents currently at the top of preference list are considered + * to have the answer NO right away, before doing any other constraint checks. + * 6. When you have hoistable answer for each dependency of a node you then build + * a list of nodes that are NOT hoistable. These are the nodes that have answer NO + * and the nodes that DEPENDS on these nodes. All the other nodes are hoistable, + * those that have answer YES and those that have answer DEPENDS, + * because they are cyclically dependent on each another + * 7. You hoist all the hoistable nodes to the current root and continue traversing + * the tree. Note, you need to track newly added nodes to the current root, + * because after you finished tree traversal you want to come back to these new nodes + * first thing and hoist everything from each of them to the current tree root. + * 8. After you have finished traversing newly hoisted current root nodes + * it means you cannot hoist anything to the current tree root and you need to pick + * the next node as current tree root and run the algorithm again + * until you run out of candidates for current tree root. + */ +type PackageName = string +export enum HoisterDependencyKind { + REGULAR, + WORKSPACE, + EXTERNAL_SOFT_LINK, +} +export type HoisterTree = { + name: PackageName + identName: PackageName + reference: string + dependencies: Set + peerNames: Set + hoistPriority?: number + dependencyKind?: HoisterDependencyKind +} +export type HoisterResult = { name: PackageName; identName: PackageName; references: Set; dependencies: Set } +type Locator = string +type AliasedLocator = string & { __aliasedLocator: true } +type Ident = string +type HoisterWorkTree = { + name: PackageName + references: Set + ident: Ident + locator: Locator + dependencies: Map + originalDependencies: Map + hoistedDependencies: Map + peerNames: ReadonlySet + decoupled: boolean + reasons: Map + isHoistBorder: boolean + hoistedFrom: Map> + hoistedTo: Map + hoistPriority: number + dependencyKind: HoisterDependencyKind +} + +/** + * Mapping which packages depend on a given package alias + ident. It is used to determine hoisting weight, + * e.g. which one among the group of packages with the same name should be hoisted. + * The package having the biggest number of parents using this package will be hoisted. + */ +type PreferenceMap = Map; dependents: Set; hoistPriority: number }> + +enum Hoistable { + YES, + NO, + DEPENDS, +} +type HoistInfo = + | { + isHoistable: Hoistable.YES + } + | { + isHoistable: Hoistable.NO + reason: string | null + } + | { + isHoistable: Hoistable.DEPENDS + dependsOn: Set + reason: string | null + } + +type ShadowedNodes = Map> + +const makeLocator = (name: string, reference: string) => `${name}@${reference}` +const makeIdent = (name: string, reference: string) => { + const hashIdx = reference.indexOf(`#`) + // Strip virtual reference part, we don't need it for hoisting purposes + const realReference = hashIdx >= 0 ? reference.substring(hashIdx + 1) : reference + return makeLocator(name, realReference) +} + +enum DebugLevel { + NONE = -1, + PERF = 0, + CHECK = 1, + REASONS = 2, + INTENSIVE_CHECK = 9, +} + +export type HoistOptions = { + /** Runs self-checks after hoisting is finished */ + check?: boolean + /** Debug level */ + debugLevel?: DebugLevel + /** Hoist borders are defined by parent node locator and its dependency name. The dependency is considered a border, nothing can be hoisted past this dependency, but dependency can be hoisted */ + hoistingLimits?: Map> +} + +type InternalHoistOptions = { + check?: boolean + debugLevel: DebugLevel + fastLookupPossible: boolean + hoistingLimits: Map> +} + +/** + * Hoists package tree. + * + * The root node of a tree must has id: '.'. + * This function does not mutate its arguments, it hoists and returns tree copy. + * + * @param tree package tree (cycles in the tree are allowed) + * + * @returns hoisted tree copy + */ +export const hoist = (tree: HoisterTree, opts: HoistOptions = {}): HoisterResult => { + const debugLevel = opts.debugLevel || Number(process.env.NM_DEBUG_LEVEL || DebugLevel.NONE) + const check = opts.check || debugLevel >= (DebugLevel.INTENSIVE_CHECK as number) + const hoistingLimits = opts.hoistingLimits || new Map() + const options: InternalHoistOptions = { check, debugLevel, hoistingLimits, fastLookupPossible: true } + let startTime: number + + if (options.debugLevel >= DebugLevel.PERF) startTime = Date.now() + + const treeCopy = cloneTree(tree, options) + + let anotherRoundNeeded = false + let round = 0 + do { + const result = hoistTo(treeCopy, [treeCopy], new Set([treeCopy.locator]), new Map(), options) + anotherRoundNeeded = result.anotherRoundNeeded || result.isGraphChanged + options.fastLookupPossible = false + round++ + } while (anotherRoundNeeded) + + if (options.debugLevel >= DebugLevel.PERF) console.log(`hoist time: ${Date.now() - startTime!}ms, rounds: ${round}`) + + if (options.debugLevel >= DebugLevel.CHECK) { + const prevTreeDump = dumpDepTree(treeCopy) + const isGraphChanged = hoistTo(treeCopy, [treeCopy], new Set([treeCopy.locator]), new Map(), options).isGraphChanged + if (isGraphChanged) throw new Error(`The hoisting result is not terminal, prev tree:\n${prevTreeDump}, next tree:\n${dumpDepTree(treeCopy)}`) + const checkLog = selfCheck(treeCopy) + if (checkLog) { + throw new Error(`${checkLog}, after hoisting finished:\n${dumpDepTree(treeCopy)}`) + } + } + + if (options.debugLevel >= DebugLevel.REASONS) console.log(dumpDepTree(treeCopy)) + + return shrinkTree(treeCopy) +} + +const getZeroRoundUsedDependencies = (rootNodePath: Array): Map => { + const rootNode = rootNodePath[rootNodePath.length - 1] + const usedDependencies = new Map() + const seenNodes = new Set() + + const addUsedDependencies = (node: HoisterWorkTree) => { + if (seenNodes.has(node)) return + seenNodes.add(node) + + for (const dep of node.hoistedDependencies.values()) usedDependencies.set(dep.name, dep) + + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + addUsedDependencies(dep) + } + } + } + + addUsedDependencies(rootNode) + + return usedDependencies +} + +const getUsedDependencies = (rootNodePath: Array): Map => { + const rootNode = rootNodePath[rootNodePath.length - 1] + const usedDependencies = new Map() + const seenNodes = new Set() + + const hiddenDependencies = new Set() + const addUsedDependencies = (node: HoisterWorkTree, hiddenDependencies: Set) => { + if (seenNodes.has(node)) return + seenNodes.add(node) + + for (const dep of node.hoistedDependencies.values()) { + if (!hiddenDependencies.has(dep.name)) { + let reachableDependency + for (const node of rootNodePath) { + reachableDependency = node.dependencies.get(dep.name) + if (reachableDependency) { + usedDependencies.set(reachableDependency.name, reachableDependency) + } + } + } + } + + const childrenHiddenDependencies = new Set() + + for (const dep of node.dependencies.values()) childrenHiddenDependencies.add(dep.name) + + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + addUsedDependencies(dep, childrenHiddenDependencies) + } + } + } + + addUsedDependencies(rootNode, hiddenDependencies) + + return usedDependencies +} + +/** + * This method clones the node and returns cloned node copy, if the node was not previously decoupled. + * + * The node is considered decoupled if there is no multiple parents to any node + * on the path from the dependency graph root up to this node. This means that there are no other + * nodes in dependency graph that somehow transitively use this node and hence node can be hoisted without + * side effects. + * + * The process of node decoupling is done by going from root node of the graph up to the node in concern + * and decoupling each node on this graph path. + * + * @param node original node + * + * @returns decoupled node + */ +const decoupleGraphNode = (parent: HoisterWorkTree, node: HoisterWorkTree): HoisterWorkTree => { + if (node.decoupled) return node + + const { + name, + references, + ident, + locator, + dependencies, + originalDependencies, + hoistedDependencies, + peerNames, + reasons, + isHoistBorder, + hoistPriority, + dependencyKind, + hoistedFrom, + hoistedTo, + } = node + // To perform node hoisting from parent node we must clone parent nodes up to the root node, + // because some other package in the tree might depend on the parent package where hoisting + // cannot be performed + const clone = { + name, + references: new Set(references), + ident, + locator, + dependencies: new Map(dependencies), + originalDependencies: new Map(originalDependencies), + hoistedDependencies: new Map(hoistedDependencies), + peerNames: new Set(peerNames), + reasons: new Map(reasons), + decoupled: true, + isHoistBorder, + hoistPriority, + dependencyKind, + hoistedFrom: new Map(hoistedFrom), + hoistedTo: new Map(hoistedTo), + } + const selfDep = clone.dependencies.get(name) + if (selfDep && selfDep.ident == clone.ident) + // Update self-reference + clone.dependencies.set(name, clone) + + parent.dependencies.set(clone.name, clone) + + return clone +} + +/** + * Builds a map of most preferred packages that might be hoisted to the root node. + * + * The values in the map are idents sorted by preference from most preferred to less preferred. + * If the root node has already some version of a package, the value array will contain only + * one element, since it is not possible for other versions of a package to be hoisted. + * + * @param rootNode root node + * @param preferenceMap preference map + */ +const getHoistIdentMap = (rootNode: HoisterWorkTree, preferenceMap: PreferenceMap): Map> => { + const identMap = new Map>([[rootNode.name, [rootNode.ident]]]) + + for (const dep of rootNode.dependencies.values()) { + if (!rootNode.peerNames.has(dep.name)) { + identMap.set(dep.name, [dep.ident]) + } + } + + const keyList = Array.from(preferenceMap.keys()) + keyList.sort((key1, key2) => { + const entry1 = preferenceMap.get(key1)! + const entry2 = preferenceMap.get(key2)! + if (entry2.hoistPriority !== entry1.hoistPriority) { + return entry2.hoistPriority - entry1.hoistPriority + } else { + const entry1Usages = entry1.dependents.size + entry1.peerDependents.size + const entry2Usages = entry2.dependents.size + entry2.peerDependents.size + return entry2Usages - entry1Usages + } + }) + + for (const key of keyList) { + const name = key.substring(0, key.indexOf(`@`, 1)) + const ident = key.substring(name.length + 1) + if (!rootNode.peerNames.has(name)) { + let idents = identMap.get(name) + if (!idents) { + idents = [] + identMap.set(name, idents) + } + if (idents.indexOf(ident) < 0) { + idents.push(ident) + } + } + } + + return identMap +} + +/** + * Gets regular node dependencies only and sorts them in the order so that + * peer dependencies come before the dependency that rely on them. + * + * @param node graph node + * @returns sorted regular dependencies + */ +const getSortedRegularDependencies = (node: HoisterWorkTree): Set => { + const dependencies: Set = new Set() + + const addDep = (dep: HoisterWorkTree, seenDeps = new Set()) => { + if (seenDeps.has(dep)) return + seenDeps.add(dep) + + for (const peerName of dep.peerNames) { + if (!node.peerNames.has(peerName)) { + const peerDep = node.dependencies.get(peerName) + if (peerDep && !dependencies.has(peerDep)) { + addDep(peerDep, seenDeps) + } + } + } + dependencies.add(dep) + } + + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + addDep(dep) + } + } + + return dependencies +} + +/** + * Performs hoisting all the dependencies down the tree to the root node. + * + * The algorithm used here reduces dependency graph by deduplicating + * instances of the packages while keeping: + * 1. Regular dependency promise: the package should require the exact version of the dependency + * that was declared in its `package.json` + * 2. Peer dependency promise: the package and its direct parent package + * must use the same instance of the peer dependency + * + * The regular and peer dependency promises are kept while performing transform + * on tree branches of packages at a time: + * `root package` -> `parent package 1` ... `parent package n` -> `dependency` + * We check wether we can hoist `dependency` to `root package`, this boils down basically + * to checking: + * 1. Wether `root package` does not depend on other version of `dependency` + * 2. Wether all the peer dependencies of a `dependency` had already been hoisted from all `parent packages` + * + * If many versions of the `dependency` can be hoisted to the `root package` we choose the most used + * `dependency` version in the project among them. + * + * This function mutates the tree. + * + * @param tree package dependencies graph + * @param rootNode root node to hoist to + * @param rootNodePath root node path in the tree + * @param rootNodePathLocators a set of locators for nodes that lead from the top of the tree up to root node + * @param options hoisting options + */ +const hoistTo = ( + tree: HoisterWorkTree, + rootNodePath: Array, + rootNodePathLocators: Set, + parentShadowedNodes: ShadowedNodes, + options: InternalHoistOptions, + seenNodes: Set = new Set() +): { anotherRoundNeeded: boolean; isGraphChanged: boolean } => { + const rootNode = rootNodePath[rootNodePath.length - 1] + if (seenNodes.has(rootNode)) return { anotherRoundNeeded: false, isGraphChanged: false } + seenNodes.add(rootNode) + + const preferenceMap = buildPreferenceMap(rootNode) + + const hoistIdentMap = getHoistIdentMap(rootNode, preferenceMap) + + const usedDependencies = tree == rootNode ? new Map() : options.fastLookupPossible ? getZeroRoundUsedDependencies(rootNodePath) : getUsedDependencies(rootNodePath) + + let wasStateChanged + + let anotherRoundNeeded = false + let isGraphChanged = false + + const hoistIdents = new Map(Array.from(hoistIdentMap.entries()).map(([k, v]) => [k, v[0]])) + const shadowedNodes: ShadowedNodes = new Map() + do { + const result = hoistGraph(tree, rootNodePath, rootNodePathLocators, usedDependencies, hoistIdents, hoistIdentMap, parentShadowedNodes, shadowedNodes, options) + if (result.isGraphChanged) isGraphChanged = true + if (result.anotherRoundNeeded) anotherRoundNeeded = true + + wasStateChanged = false + for (const [name, idents] of hoistIdentMap) { + if (idents.length > 1 && !rootNode.dependencies.has(name)) { + hoistIdents.delete(name) + idents.shift() + hoistIdents.set(name, idents[0]) + wasStateChanged = true + } + } + } while (wasStateChanged) + + for (const dependency of rootNode.dependencies.values()) { + if (!rootNode.peerNames.has(dependency.name) && !rootNodePathLocators.has(dependency.locator)) { + rootNodePathLocators.add(dependency.locator) + const result = hoistTo(tree, [...rootNodePath, dependency], rootNodePathLocators, shadowedNodes, options) + if (result.isGraphChanged) isGraphChanged = true + if (result.anotherRoundNeeded) anotherRoundNeeded = true + + rootNodePathLocators.delete(dependency.locator) + } + } + + return { anotherRoundNeeded, isGraphChanged } +} + +const hasUnhoistedDependencies = (node: HoisterWorkTree): boolean => { + for (const [subName, subDependency] of node.dependencies) { + if (!node.peerNames.has(subName) && subDependency.ident !== node.ident) { + return true + } + } + return false +} + +const getNodeHoistInfo = ( + rootNode: HoisterWorkTree, + rootNodePathLocators: Set, + nodePath: Array, + node: HoisterWorkTree, + usedDependencies: Map, + hoistIdents: Map, + hoistIdentMap: Map>, + shadowedNodes: ShadowedNodes, + { outputReason, fastLookupPossible }: { outputReason: boolean; fastLookupPossible: boolean } +): HoistInfo => { + let reasonRoot + let reason: string | null = null + let dependsOn: Set | null = new Set() + if (outputReason) + reasonRoot = `${Array.from(rootNodePathLocators) + .map(x => prettyPrintLocator(x)) + .join(`→`)}` + + const parentNode = nodePath[nodePath.length - 1] + // We cannot hoist self-references + const isSelfReference = node.ident === parentNode.ident + let isHoistable = !isSelfReference + if (outputReason && !isHoistable) reason = `- self-reference` + + if (isHoistable) { + isHoistable = node.dependencyKind !== HoisterDependencyKind.WORKSPACE + if (outputReason && !isHoistable) { + reason = `- workspace` + } + } + + if (isHoistable && node.dependencyKind === HoisterDependencyKind.EXTERNAL_SOFT_LINK) { + isHoistable = !hasUnhoistedDependencies(node) + if (outputReason && !isHoistable) { + reason = `- external soft link with unhoisted dependencies` + } + } + + if (isHoistable) { + isHoistable = !rootNode.peerNames.has(node.name) + if (outputReason && !isHoistable) { + reason = `- cannot shadow peer: ${prettyPrintLocator(rootNode.originalDependencies.get(node.name)!.locator)} at ${reasonRoot}` + } + } + + if (isHoistable) { + let isNameAvailable = false + const usedDep = usedDependencies.get(node.name) + isNameAvailable = !usedDep || usedDep.ident === node.ident + if (outputReason && !isNameAvailable) reason = `- filled by: ${prettyPrintLocator(usedDep!.locator)} at ${reasonRoot}` + if (isNameAvailable) { + for (let idx = nodePath.length - 1; idx >= 1; idx--) { + const parent = nodePath[idx] + const parentDep = parent.dependencies.get(node.name) + if (parentDep && parentDep.ident !== node.ident) { + isNameAvailable = false + let shadowedNames = shadowedNodes.get(parentNode) + if (!shadowedNames) { + shadowedNames = new Set() + shadowedNodes.set(parentNode, shadowedNames) + } + shadowedNames.add(node.name) + if (outputReason) + reason = `- filled by ${prettyPrintLocator(parentDep.locator)} at ${nodePath + .slice(0, idx) + .map(x => prettyPrintLocator(x.locator)) + .join(`→`)}` + break + } + } + } + + isHoistable = isNameAvailable + } + + if (isHoistable) { + const hoistedIdent = hoistIdents.get(node.name) + isHoistable = hoistedIdent === node.ident + if (outputReason && !isHoistable) { + reason = `- filled by: ${prettyPrintLocator(hoistIdentMap.get(node.name)![0])} at ${reasonRoot}` + } + } + + if (isHoistable) { + let arePeerDepsSatisfied = true + const checkList = new Set(node.peerNames) + for (let idx = nodePath.length - 1; idx >= 1; idx--) { + const parent = nodePath[idx] + for (const name of checkList) { + if (parent.peerNames.has(name) && parent.originalDependencies.has(name)) continue + + const parentDepNode = parent.dependencies.get(name) + if (parentDepNode && rootNode.dependencies.get(name) !== parentDepNode) { + if (idx === nodePath.length - 1) { + dependsOn!.add(parentDepNode) + } else { + dependsOn = null + arePeerDepsSatisfied = false + if (outputReason) { + reason = `- peer dependency ${prettyPrintLocator(parentDepNode.locator)} from parent ${prettyPrintLocator(parent.locator)} was not hoisted to ${reasonRoot}` + } + } + } + checkList.delete(name) + } + if (!arePeerDepsSatisfied) { + break + } + } + isHoistable = arePeerDepsSatisfied + } + + if (isHoistable && !fastLookupPossible) { + for (const origDep of node.hoistedDependencies.values()) { + const usedDep = usedDependencies.get(origDep.name) || rootNode.dependencies.get(origDep.name) + if (!usedDep || origDep.ident !== usedDep.ident) { + isHoistable = false + if (outputReason) reason = `- previously hoisted dependency mismatch, needed: ${prettyPrintLocator(origDep.locator)}, available: ${prettyPrintLocator(usedDep?.locator)}` + + break + } + } + } + + if (dependsOn !== null && dependsOn.size > 0) { + return { isHoistable: Hoistable.DEPENDS, dependsOn, reason } + } else { + return { isHoistable: isHoistable ? Hoistable.YES : Hoistable.NO, reason } + } +} + +const getAliasedLocator = (node: HoisterWorkTree): AliasedLocator => `${node.name}@${node.locator}` as AliasedLocator + +/** + * Performs actual graph transformation, by hoisting packages to the root node. + * + * @param tree dependency tree + * @param rootNodePath root node path in the tree + * @param rootNodePathLocators a set of locators for nodes that lead from the top of the tree up to root node + * @param usedDependencies map of dependency nodes from parents of root node used by root node and its children via parent lookup + * @param hoistIdents idents that should be attempted to be hoisted to the root node + */ +const hoistGraph = ( + tree: HoisterWorkTree, + rootNodePath: Array, + rootNodePathLocators: Set, + usedDependencies: Map, + hoistIdents: Map, + hoistIdentMap: Map>, + parentShadowedNodes: ShadowedNodes, + shadowedNodes: ShadowedNodes, + options: InternalHoistOptions +): { anotherRoundNeeded: boolean; isGraphChanged: boolean } => { + const rootNode = rootNodePath[rootNodePath.length - 1] + const seenNodes = new Set() + let anotherRoundNeeded = false + let isGraphChanged = false + + const hoistNodeDependencies = ( + nodePath: Array, + locatorPath: Array, + aliasedLocatorPath: Array, + parentNode: HoisterWorkTree, + newNodes: Set + ) => { + if (seenNodes.has(parentNode)) return + const nextLocatorPath = [...locatorPath, getAliasedLocator(parentNode)] + const nextAliasedLocatorPath = [...aliasedLocatorPath, getAliasedLocator(parentNode)] + + const dependantTree = new Map>() + const hoistInfos = new Map() + for (const subDependency of getSortedRegularDependencies(parentNode)) { + const hoistInfo = getNodeHoistInfo( + rootNode, + rootNodePathLocators, + [rootNode, ...nodePath, parentNode], + subDependency, + usedDependencies, + hoistIdents, + hoistIdentMap, + shadowedNodes, + { outputReason: options.debugLevel >= DebugLevel.REASONS, fastLookupPossible: options.fastLookupPossible } + ) + + hoistInfos.set(subDependency, hoistInfo) + if (hoistInfo.isHoistable === Hoistable.DEPENDS) { + for (const node of hoistInfo.dependsOn) { + const nodeDependants = dependantTree.get(node.name) || new Set() + nodeDependants.add(subDependency.name) + dependantTree.set(node.name, nodeDependants) + } + } + } + + const unhoistableNodes = new Set() + const addUnhoistableNode = (node: HoisterWorkTree, hoistInfo: HoistInfo, reason: string) => { + if (!unhoistableNodes.has(node)) { + unhoistableNodes.add(node) + hoistInfos.set(node, { isHoistable: Hoistable.NO, reason }) + for (const dependantName of dependantTree.get(node.name) || []) { + addUnhoistableNode( + parentNode.dependencies.get(dependantName)!, + hoistInfo, + options.debugLevel >= DebugLevel.REASONS + ? `- peer dependency ${prettyPrintLocator(node.locator)} from parent ${prettyPrintLocator(parentNode.locator)} was not hoisted` + : `` + ) + } + } + } + + for (const [node, hoistInfo] of hoistInfos) if (hoistInfo.isHoistable === Hoistable.NO) addUnhoistableNode(node, hoistInfo, hoistInfo.reason!) + + let wereNodesHoisted = false + for (const node of hoistInfos.keys()) { + if (!unhoistableNodes.has(node)) { + isGraphChanged = true + const shadowedNames = parentShadowedNodes.get(parentNode) + if (shadowedNames && shadowedNames.has(node.name)) anotherRoundNeeded = true + + wereNodesHoisted = true + parentNode.dependencies.delete(node.name) + parentNode.hoistedDependencies.set(node.name, node) + parentNode.reasons.delete(node.name) + + const hoistedNode = rootNode.dependencies.get(node.name) + if (options.debugLevel >= DebugLevel.REASONS) { + const hoistedFrom = Array.from(locatorPath) + .concat([parentNode.locator]) + .map(x => prettyPrintLocator(x)) + .join(`→`) + let hoistedFromArray = rootNode.hoistedFrom.get(node.name) + if (!hoistedFromArray) { + hoistedFromArray = [] + rootNode.hoistedFrom.set(node.name, hoistedFromArray) + } + hoistedFromArray.push(hoistedFrom) + + parentNode.hoistedTo.set( + node.name, + Array.from(rootNodePath) + .map(x => prettyPrintLocator(x.locator)) + .join(`→`) + ) + } + // Add hoisted node to root node, in case it is not already there + if (!hoistedNode) { + // Avoid adding other version of root node to itself + if (rootNode.ident !== node.ident) { + rootNode.dependencies.set(node.name, node) + newNodes.add(node) + } + } else { + for (const reference of node.references) { + hoistedNode.references.add(reference) + } + } + } + } + + if (parentNode.dependencyKind === HoisterDependencyKind.EXTERNAL_SOFT_LINK && wereNodesHoisted) anotherRoundNeeded = true + + if (options.check) { + const checkLog = selfCheck(tree) + if (checkLog) { + throw new Error( + `${checkLog}, after hoisting dependencies of ${[rootNode, ...nodePath, parentNode].map(x => prettyPrintLocator(x.locator)).join(`→`)}:\n${dumpDepTree(tree)}` + ) + } + } + + const children = getSortedRegularDependencies(parentNode) + for (const node of children) { + if (unhoistableNodes.has(node)) { + const hoistInfo = hoistInfos.get(node)! + const hoistableIdent = hoistIdents.get(node.name) + if ((hoistableIdent === node.ident || !parentNode.reasons.has(node.name)) && hoistInfo.isHoistable !== Hoistable.YES) parentNode.reasons.set(node.name, hoistInfo.reason!) + + if (!node.isHoistBorder && nextAliasedLocatorPath.indexOf(getAliasedLocator(node)) < 0) { + seenNodes.add(parentNode) + const decoupledNode = decoupleGraphNode(parentNode, node) + + hoistNodeDependencies([...nodePath, parentNode], nextLocatorPath, nextAliasedLocatorPath, decoupledNode, nextNewNodes) + + seenNodes.delete(parentNode) + } + } + } + } + + let newNodes + let nextNewNodes = new Set(getSortedRegularDependencies(rootNode)) + const aliasedRootNodePathLocators = Array.from(rootNodePath).map(x => getAliasedLocator(x)) + do { + newNodes = nextNewNodes + nextNewNodes = new Set() + for (const dep of newNodes) { + if (dep.locator === rootNode.locator || dep.isHoistBorder) continue + const decoupledDependency = decoupleGraphNode(rootNode, dep) + + hoistNodeDependencies([], Array.from(rootNodePathLocators), aliasedRootNodePathLocators, decoupledDependency, nextNewNodes) + } + } while (nextNewNodes.size > 0) + + return { anotherRoundNeeded, isGraphChanged } +} + +const selfCheck = (tree: HoisterWorkTree): string => { + const log: Array = [] + + const seenNodes = new Set() + const parents = new Set() + + const checkNode = (node: HoisterWorkTree, parentDeps: Map, parent: HoisterWorkTree) => { + if (seenNodes.has(node)) return + seenNodes.add(node) + + if (parents.has(node)) return + + const dependencies = new Map(parentDeps) + for (const dep of node.dependencies.values()) if (!node.peerNames.has(dep.name)) dependencies.set(dep.name, dep) + + for (const origDep of node.originalDependencies.values()) { + const dep = dependencies.get(origDep.name) + const prettyPrintTreePath = () => + `${Array.from(parents) + .concat([node]) + .map(x => prettyPrintLocator(x.locator)) + .join(`→`)}` + if (node.peerNames.has(origDep.name)) { + const parentDep = parentDeps.get(origDep.name) + if (parentDep !== dep || !parentDep || parentDep.ident !== origDep.ident) { + log.push(`${prettyPrintTreePath()} - broken peer promise: expected ${origDep.ident} but found ${parentDep ? parentDep.ident : parentDep}`) + } + } else { + const hoistedFrom = parent.hoistedFrom.get(node.name) + const originalHoistedTo = node.hoistedTo.get(origDep.name) + const prettyHoistedFrom = `${hoistedFrom ? ` hoisted from ${hoistedFrom.join(`, `)}` : ``}` + const prettyOriginalHoistedTo = `${originalHoistedTo ? ` hoisted to ${originalHoistedTo}` : ``}` + const prettyNodePath = `${prettyPrintTreePath()}${prettyHoistedFrom}` + if (!dep) { + log.push(`${prettyNodePath} - broken require promise: no required dependency ${origDep.name}${prettyOriginalHoistedTo} found`) + } else if (dep.ident !== origDep.ident) { + log.push(`${prettyNodePath} - broken require promise for ${origDep.name}${prettyOriginalHoistedTo}: expected ${origDep.ident}, but found: ${dep.ident}`) + } + } + } + + parents.add(node) + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + checkNode(dep, dependencies, node) + } + } + parents.delete(node) + } + + checkNode(tree, tree.dependencies, tree) + + return log.join(`\n`) +} + +/** + * Creates a clone of package tree with extra fields used for hoisting purposes. + * + * @param tree package tree clone + */ +const cloneTree = (tree: HoisterTree, options: InternalHoistOptions): HoisterWorkTree => { + const { identName, name, reference, peerNames } = tree + const treeCopy: HoisterWorkTree = { + name, + references: new Set([reference]), + locator: makeLocator(identName, reference), + ident: makeIdent(identName, reference), + dependencies: new Map(), + originalDependencies: new Map(), + hoistedDependencies: new Map(), + peerNames: new Set(peerNames), + reasons: new Map(), + decoupled: true, + isHoistBorder: true, + hoistPriority: 0, + dependencyKind: HoisterDependencyKind.WORKSPACE, + hoistedFrom: new Map(), + hoistedTo: new Map(), + } + + const seenNodes = new Map([[tree, treeCopy]]) + + const addNode = (node: HoisterTree, parentNode: HoisterWorkTree) => { + let workNode = seenNodes.get(node) + const isSeen = !!workNode + if (!workNode) { + const { name, identName, reference, peerNames, hoistPriority, dependencyKind } = node + const dependenciesNmHoistingLimits = options.hoistingLimits.get(parentNode.locator) + workNode = { + name, + references: new Set([reference]), + locator: makeLocator(identName, reference), + ident: makeIdent(identName, reference), + dependencies: new Map(), + originalDependencies: new Map(), + hoistedDependencies: new Map(), + peerNames: new Set(peerNames), + reasons: new Map(), + decoupled: true, + isHoistBorder: dependenciesNmHoistingLimits ? dependenciesNmHoistingLimits.has(name) : false, + hoistPriority: hoistPriority || 0, + dependencyKind: dependencyKind || HoisterDependencyKind.REGULAR, + hoistedFrom: new Map(), + hoistedTo: new Map(), + } + seenNodes.set(node, workNode) + } + + parentNode.dependencies.set(node.name, workNode) + parentNode.originalDependencies.set(node.name, workNode) + + if (!isSeen) { + for (const dep of node.dependencies) { + addNode(dep, workNode) + } + } else { + const seenCoupledNodes = new Set() + + const markNodeCoupled = (node: HoisterWorkTree) => { + if (seenCoupledNodes.has(node)) return + seenCoupledNodes.add(node) + node.decoupled = false + + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + markNodeCoupled(dep) + } + } + } + + markNodeCoupled(workNode) + } + } + + for (const dep of tree.dependencies) addNode(dep, treeCopy) + + return treeCopy +} + +const getIdentName = (locator: Locator) => locator.substring(0, locator.indexOf(`@`, 1)) + +/** + * Creates a clone of hoisted package tree with extra fields removed + * + * @param tree stripped down hoisted package tree clone + */ +const shrinkTree = (tree: HoisterWorkTree): HoisterResult => { + const treeCopy: HoisterResult = { + name: tree.name, + identName: getIdentName(tree.locator), + references: new Set(tree.references), + dependencies: new Set(), + } + + const seenNodes = new Set([tree]) + + const addNode = (node: HoisterWorkTree, parentWorkNode: HoisterWorkTree, parentNode: HoisterResult) => { + const isSeen = seenNodes.has(node) + + let resultNode: HoisterResult + if (parentWorkNode === node) { + resultNode = parentNode + } else { + const { name, references, locator } = node + resultNode = { + name, + identName: getIdentName(locator), + references, + dependencies: new Set(), + } + } + parentNode.dependencies.add(resultNode) + + if (!isSeen) { + seenNodes.add(node) + for (const dep of node.dependencies.values()) { + if (!node.peerNames.has(dep.name)) { + addNode(dep, node, resultNode) + } + } + seenNodes.delete(node) + } + } + + for (const dep of tree.dependencies.values()) addNode(dep, tree, treeCopy) + + return treeCopy +} + +/** + * Builds mapping, where key is an alias + dependent package ident and the value is the list of + * parent package idents who depend on this package. + * + * @param rootNode package tree root node + * + * @returns preference map + */ +const buildPreferenceMap = (rootNode: HoisterWorkTree): PreferenceMap => { + const preferenceMap: PreferenceMap = new Map() + + const seenNodes = new Set([rootNode]) + const getPreferenceKey = (node: HoisterWorkTree) => `${node.name}@${node.ident}` + + const getOrCreatePreferenceEntry = (node: HoisterWorkTree) => { + const key = getPreferenceKey(node) + let entry = preferenceMap.get(key) + if (!entry) { + entry = { dependents: new Set(), peerDependents: new Set(), hoistPriority: 0 } + preferenceMap.set(key, entry) + } + return entry + } + + const addDependent = (dependent: HoisterWorkTree, node: HoisterWorkTree) => { + const isSeen = !!seenNodes.has(node) + + const entry = getOrCreatePreferenceEntry(node) + entry.dependents.add(dependent.ident) + + if (!isSeen) { + seenNodes.add(node) + for (const dep of node.dependencies.values()) { + const entry = getOrCreatePreferenceEntry(dep) + entry.hoistPriority = Math.max(entry.hoistPriority, dep.hoistPriority) + if (node.peerNames.has(dep.name)) { + entry.peerDependents.add(node.ident) + } else { + addDependent(node, dep) + } + } + } + } + + for (const dep of rootNode.dependencies.values()) if (!rootNode.peerNames.has(dep.name)) addDependent(rootNode, dep) + + return preferenceMap +} + +const prettyPrintLocator = (locator?: Locator) => { + if (!locator) return `none` + + const idx = locator.indexOf(`@`, 1) + let name = locator.substring(0, idx) + if (name.endsWith(`$wsroot$`)) name = `wh:${name.replace(`$wsroot$`, ``)}` + const reference = locator.substring(idx + 1) + if (reference === `workspace:.`) { + return `.` + } else if (!reference) { + return `${name}` + } else { + let version = (reference.indexOf(`#`) > 0 ? reference.split(`#`)[1] : reference).replace(`npm:`, ``) + if (reference.startsWith(`virtual`)) name = `v:${name}` + if (version.startsWith(`workspace`)) { + name = `w:${name}` + version = `` + } + + return `${name}${version ? `@${version}` : ``}` + } +} + +const MAX_NODES_TO_DUMP = 50000 + +/** + * Pretty-prints dependency tree in the `yarn why`-like format + * + * The function is used for troubleshooting purposes only. + * + * @param pkg node_modules tree + * + * @returns sorted node_modules tree + */ + +const dumpDepTree = (tree: HoisterWorkTree) => { + let nodeCount = 0 + const dumpPackage = (pkg: HoisterWorkTree, parents: Set, prefix = ``): string => { + if (nodeCount > MAX_NODES_TO_DUMP || parents.has(pkg)) return `` + + nodeCount++ + const dependencies = Array.from(pkg.dependencies.values()).sort((n1, n2) => { + if (n1.name === n2.name) { + return 0 + } else { + return n1.name > n2.name ? 1 : -1 + } + }) + + let str = `` + parents.add(pkg) + for (let idx = 0; idx < dependencies.length; idx++) { + const dep = dependencies[idx] + if (!pkg.peerNames.has(dep.name) && dep !== pkg) { + const reason = pkg.reasons.get(dep.name) + const identName = getIdentName(dep.locator) + str += `${prefix}${idx < dependencies.length - 1 ? `├─` : `└─`}${(parents.has(dep) ? `>` : ``) + (identName !== dep.name ? `a:${dep.name}:` : ``) + prettyPrintLocator(dep.locator) + (reason ? ` ${reason}` : ``)}\n` + str += dumpPackage(dep, parents, `${prefix}${idx < dependencies.length - 1 ? `│ ` : ` `}`) + } + } + parents.delete(pkg) + return str + } + + const treeDump = dumpPackage(tree, new Set()) + + return treeDump + (nodeCount > MAX_NODES_TO_DUMP ? `\nTree is too large, part of the tree has been dunped\n` : ``) +} diff --git a/packages/app-builder-lib/src/node-module-collector/index.ts b/packages/app-builder-lib/src/node-module-collector/index.ts new file mode 100644 index 00000000000..4bcf2cc9f3e --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/index.ts @@ -0,0 +1,26 @@ +import { NpmNodeModulesCollector } from "./npmNodeModulesCollector" +import { PnpmNodeModulesCollector } from "./pnpmNodeModulesCollector" +import { YarnNodeModulesCollector } from "./yarnNodeModulesCollector" +import { detect, PM, getPackageManagerVersion } from "./packageManager" +import { NodeModuleInfo } from "./types" + +async function getCollectorByPackageManager(rootDir: string) { + const manager: PM = await detect({ cwd: rootDir }) + switch (manager) { + case "npm": + return new NpmNodeModulesCollector(rootDir) + case "pnpm": + return new PnpmNodeModulesCollector(rootDir) + case "yarn": + return new YarnNodeModulesCollector(rootDir) + default: + return new NpmNodeModulesCollector(rootDir) + } +} + +export async function getNodeModules(rootDir: string): Promise { + const collector = await getCollectorByPackageManager(rootDir) + return collector.getNodeModules() +} + +export { detect, getPackageManagerVersion, PM } diff --git a/packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts b/packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts new file mode 100644 index 00000000000..7a1666cf066 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts @@ -0,0 +1,135 @@ +import { hoist, type HoisterTree, type HoisterResult } from "./hoist" +import * as path from "path" +import * as fs from "fs" +import { NodeModuleInfo, DependencyTree, DependencyGraph } from "./types" +import { exec, log } from "builder-util" + +export abstract class NodeModulesCollector { + private nodeModules: NodeModuleInfo[] + protected dependencyPathMap: Map + + constructor(private readonly rootDir: string) { + this.dependencyPathMap = new Map() + this.nodeModules = [] + } + + private transToHoisterTree(obj: DependencyGraph, key: string = `.`, nodes: Map = new Map()): HoisterTree { + let node = nodes.get(key) + const name = key.match(/@?[^@]+/)![0] + if (!node) { + node = { + name, + identName: name, + reference: key.match(/@?[^@]+@?(.+)?/)![1] || ``, + dependencies: new Set(), + peerNames: new Set([]), + } + nodes.set(key, node) + + for (const dep of (obj[key] || {}).dependencies || []) { + node.dependencies.add(this.transToHoisterTree(obj, dep, nodes)) + } + } + return node + } + + private resolvePath(filePath: string) { + try { + const stats = fs.lstatSync(filePath) + if (stats.isSymbolicLink()) { + return fs.realpathSync(filePath) + } else { + return filePath + } + } catch (error: any) { + log.debug({ message: error.message || error.stack }, "error resolving path") + return filePath + } + } + + public convertToDependencyGraph(tree: DependencyTree): DependencyGraph { + const result: DependencyGraph = { ".": {} } + + const flatten = (node: DependencyTree, parentKey = ".") => { + const dependencies = node.dependencies || {} + + for (const [key, value] of Object.entries(dependencies)) { + // Skip empty dependencies(like some optionalDependencies) + if (Object.keys(value).length === 0) { + continue + } + const version = value.version || "" + const newKey = `${key}@${version}` + this.dependencyPathMap.set(newKey, path.normalize(this.resolvePath(value.path))) + if (!result[parentKey]?.dependencies) { + result[parentKey] = { dependencies: [] } + } + result[parentKey].dependencies!.push(newKey) + flatten(value, newKey) + } + } + + flatten(tree) + return result + } + + abstract getCommand(): string + abstract getArgs(): string[] + + protected async getDependenciesTree(): Promise { + const command = this.getCommand() + const args = this.getArgs() + const dependencies = await exec(command, args, { + cwd: this.rootDir, + shell: true, + }) + const dependencyTree: DependencyTree | DependencyTree[] = JSON.parse(dependencies) + return Array.isArray(dependencyTree) ? dependencyTree[0] : dependencyTree + } + + private _getNodeModules(dependencies: Set, result: NodeModuleInfo[]) { + if (dependencies.size === 0) { + return + } + + for (const d of dependencies.values()) { + const reference = [...d.references][0] + const p = this.dependencyPathMap.get(`${d.name}@${reference}`) + if (p === undefined) { + log.debug({ name: d.name, reference }, "cannot find path for dependency") + continue + } + const node: NodeModuleInfo = { + name: d.name, + version: reference, + dir: p, + } + result.push(node) + if (d.dependencies.size > 0) { + node["dependencies"] = [] + this._getNodeModules(d.dependencies, node["dependencies"]) + } + } + result.sort((a, b) => a.name.localeCompare(b.name)) + } + + private getTreeFromWorkspaces(tree: DependencyTree): DependencyTree { + if (tree.workspaces && tree.dependencies) { + for (const [key, value] of Object.entries(tree.dependencies)) { + if (this.rootDir.endsWith(path.normalize(key))) { + return value + } + } + } + return tree + } + + public async getNodeModules(): Promise { + const tree = await this.getDependenciesTree() + const realTree = this.getTreeFromWorkspaces(tree) + const dependencyGraph = this.convertToDependencyGraph(realTree) + const hoisterResult = hoist(this.transToHoisterTree(dependencyGraph), { check: true }) + this._getNodeModules(hoisterResult.dependencies, this.nodeModules) + return this.nodeModules + } +} diff --git a/packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts b/packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts new file mode 100644 index 00000000000..2e780e460c8 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts @@ -0,0 +1,15 @@ +import { NodeModulesCollector } from "./nodeModulesCollector" + +export class NpmNodeModulesCollector extends NodeModulesCollector { + constructor(rootDir: string) { + super(rootDir) + } + + getCommand(): string { + return process.platform === "win32" ? "npm.cmd" : "npm" + } + + getArgs(): string[] { + return ["list", "--omit", "dev", "-a", "--json", "--long", "--silent"] + } +} diff --git a/packages/app-builder-lib/src/node-module-collector/packageManager.ts b/packages/app-builder-lib/src/node-module-collector/packageManager.ts new file mode 100644 index 00000000000..0d9f376ba28 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/packageManager.ts @@ -0,0 +1,95 @@ +// copy from https://github.com/egoist/detect-package-manager/blob/main/src/index.ts +// and merge https://github.com/egoist/detect-package-manager/pull/9 to support Monorepo +import { resolve, dirname } from "path" +import { exec, exists } from "builder-util" + +export type PM = "npm" | "yarn" | "pnpm" | "bun" + +const cache = new Map() +const globalInstallationCache = new Map() +const lockfileCache = new Map() + +/** + * Check if a global pm is available + */ +function hasGlobalInstallation(pm: PM): Promise { + const key = `has_global_${pm}` + if (globalInstallationCache.has(key)) { + return Promise.resolve(globalInstallationCache.get(key)!) + } + + return exec(pm, ["--version"]) + .then(res => { + return /^\d+.\d+.\d+$/.test(res) + }) + .then(value => { + globalInstallationCache.set(key, value) + return value + }) + .catch(() => false) +} + +function getTypeofLockFile(cwd = process.cwd()): Promise { + const key = `lockfile_${cwd}` + if (lockfileCache.has(key)) { + return Promise.resolve(lockfileCache.get(key)!) + } + + return Promise.all([ + exists(resolve(cwd, "yarn.lock")), + exists(resolve(cwd, "package-lock.json")), + exists(resolve(cwd, "pnpm-lock.yaml")), + exists(resolve(cwd, "bun.lockb")), + ]).then(([isYarn, _, isPnpm, isBun]) => { + let value: PM + + if (isYarn) { + value = "yarn" + } else if (isPnpm) { + value = "pnpm" + } else if (isBun) { + value = "bun" + } else { + value = "npm" + } + + cache.set(key, value) + return value + }) +} + +export const detect = async ({ cwd, includeGlobalBun }: { cwd?: string; includeGlobalBun?: boolean } = {}) => { + let type = await getTypeofLockFile(cwd) + if (type) { + return type + } + + let tmpCwd = cwd || "." + for (let i = 1; i <= 5; i++) { + tmpCwd = dirname(tmpCwd) + type = await getTypeofLockFile(tmpCwd) + if (type) { + return type + } + } + + if (await hasGlobalInstallation("yarn")) { + return "yarn" + } + if (await hasGlobalInstallation("pnpm")) { + return "yarn" + } + + if (includeGlobalBun && (await hasGlobalInstallation("bun"))) { + return "bun" + } + return "npm" +} + +export function getPackageManagerVersion(pm: PM) { + return exec(pm, ["--version"]).then(res => res.trim()) +} + +export function clearCache() { + return cache.clear() +} diff --git a/packages/app-builder-lib/src/node-module-collector/pnpmNodeModulesCollector.ts b/packages/app-builder-lib/src/node-module-collector/pnpmNodeModulesCollector.ts new file mode 100644 index 00000000000..8ae627c1d47 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/pnpmNodeModulesCollector.ts @@ -0,0 +1,15 @@ +import { NodeModulesCollector } from "./nodeModulesCollector" + +export class PnpmNodeModulesCollector extends NodeModulesCollector { + constructor(rootDir: string) { + super(rootDir) + } + + getCommand(): string { + return process.platform === "win32" ? "pnpm.cmd" : "pnpm" + } + + getArgs(): string[] { + return ["list", "--prod", "--json", "--long", "--depth", "Infinity"] + } +} diff --git a/packages/app-builder-lib/src/node-module-collector/types.ts b/packages/app-builder-lib/src/node-module-collector/types.ts new file mode 100644 index 00000000000..475d7b349b9 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/types.ts @@ -0,0 +1,25 @@ +export interface NodeModuleInfo { + name: string + version: string + dir: string + dependencies?: Array +} + +export interface DependencyTree { + readonly version?: string + readonly name?: string + readonly from?: string + readonly workspaces?: string[] + readonly path: string + dependencies: { + [packageName: string]: DependencyTree + } +} + +export interface DependencyGraph { + [packageNameAndVersion: string]: PackageDependencies +} + +interface PackageDependencies { + dependencies?: string[] +} diff --git a/packages/app-builder-lib/src/node-module-collector/yarnNodeModulesCollector.ts b/packages/app-builder-lib/src/node-module-collector/yarnNodeModulesCollector.ts new file mode 100644 index 00000000000..e6b5e1bb3b4 --- /dev/null +++ b/packages/app-builder-lib/src/node-module-collector/yarnNodeModulesCollector.ts @@ -0,0 +1,15 @@ +import { NodeModulesCollector } from "./nodeModulesCollector" + +export class YarnNodeModulesCollector extends NodeModulesCollector { + constructor(rootDir: string) { + super(rootDir) + } + + getCommand(): string { + return process.platform === "win32" ? "npm.cmd" : "npm" + } + + getArgs(): string[] { + return ["list", "--omit", "dev", "-a", "--json", "--long", "--silent"] + } +} diff --git a/packages/app-builder-lib/src/platformPackager.ts b/packages/app-builder-lib/src/platformPackager.ts index db7e17281be..3eb90acf396 100644 --- a/packages/app-builder-lib/src/platformPackager.ts +++ b/packages/app-builder-lib/src/platformPackager.ts @@ -384,7 +384,7 @@ export abstract class PlatformPackager * await context.packager.addElectronFuses(context, { ... }) * ``` */ - public async addElectronFuses(context: AfterPackContext, fuses: FuseConfig) { + public addElectronFuses(context: AfterPackContext, fuses: FuseConfig) { const { appOutDir, electronPlatformName } = context const ext = { diff --git a/packages/app-builder-lib/src/util/appFileCopier.ts b/packages/app-builder-lib/src/util/appFileCopier.ts index c31adda3e91..9b22d96ad28 100644 --- a/packages/app-builder-lib/src/util/appFileCopier.ts +++ b/packages/app-builder-lib/src/util/appFileCopier.ts @@ -13,6 +13,7 @@ import { PlatformPackager } from "../platformPackager" import { AppFileWalker } from "./AppFileWalker" import { NodeModuleCopyHelper } from "./NodeModuleCopyHelper" import { NodeModuleInfo } from "./packageDependencies" +import { getNodeModules } from "../node-module-collector" const BOWER_COMPONENTS_PATTERN = `${path.sep}bower_components${path.sep}` /** @internal */ @@ -180,7 +181,8 @@ function validateFileSet(fileSet: ResolvedFileSet): ResolvedFileSet { /** @internal */ export async function computeNodeModuleFileSets(platformPackager: PlatformPackager, mainMatcher: FileMatcher): Promise> { - const deps = (await platformPackager.info.getNodeDependencyInfo(platformPackager.platform).value) as Array + const deps = await getNodeModules(platformPackager.info.appDir) + log.debug({ nodeModules: deps }, "collected node modules") const nodeModuleExcludedExts = getNodeModuleExcludedExts(platformPackager) // serial execution because copyNodeModules is concurrent and so, no need to increase queue/pressure @@ -195,8 +197,8 @@ export async function computeNodeModuleFileSets(platformPackager: PlatformPackag const files = await copier.collectNodeModules(dep, nodeModuleExcludedExts, path.relative(mainMatcher.to, destination)) result[index++] = validateFileSet({ src: source, destination, files, metadata: copier.metadata }) - if (dep.conflictDependency) { - for (const c of dep.conflictDependency) { + if (dep.dependencies) { + for (const c of dep.dependencies) { await collectNodeModules(c, path.join(destination, NODE_MODULES, c.name)) } } diff --git a/packages/app-builder-lib/src/util/packageDependencies.ts b/packages/app-builder-lib/src/util/packageDependencies.ts index 907c452ec59..86e11442a57 100644 --- a/packages/app-builder-lib/src/util/packageDependencies.ts +++ b/packages/app-builder-lib/src/util/packageDependencies.ts @@ -23,5 +23,5 @@ export interface NodeModuleInfo { readonly name: string readonly version: string readonly dir: string - readonly conflictDependency: Array + readonly dependencies?: Array } diff --git a/packages/app-builder-lib/src/util/yarn.ts b/packages/app-builder-lib/src/util/yarn.ts index da721ab266f..65c0366c742 100644 --- a/packages/app-builder-lib/src/util/yarn.ts +++ b/packages/app-builder-lib/src/util/yarn.ts @@ -10,6 +10,7 @@ import { getProjectRootPath } from "@electron/rebuild/lib/search-module" import { rebuild as remoteRebuild } from "./rebuild/rebuild" import { executeAppBuilderAndWriteJson } from "./appBuilder" import { RebuildMode } from "@electron/rebuild/lib/types" +import { PM, detect, getPackageManagerVersion } from "../node-module-collector" export async function installOrRebuild(config: Configuration, appDir: string, options: RebuildOptions, forceInstall = false) { const effectiveOptions: RebuildOptions = { @@ -78,13 +79,16 @@ export function getGypEnv(frameworkInfo: DesktopFrameworkInfo, platform: NodeJS. } } -function checkYarnBerry() { - const npmUserAgent = process.env["npm_config_user_agent"] || "" - const regex = /yarn\/(\d+)\./gm +async function checkYarnBerry(pm: PM) { + if (pm !== "yarn") { + return false + } + const version = await getPackageManagerVersion(pm) + if (version == null || version.split(".").length < 1) { + return false + } - const yarnVersionMatch = regex.exec(npmUserAgent) - const yarnMajorVersion = Number(yarnVersionMatch?.[1] ?? 0) - return yarnMajorVersion >= 2 + return version.split(".")[0] >= "2" } async function installDependencies(config: Configuration, appDir: string, options: RebuildOptions): Promise { @@ -93,25 +97,20 @@ async function installDependencies(config: Configuration, appDir: string, option const additionalArgs = options.additionalArgs log.info({ platform, arch, appDir }, `installing production dependencies`) - let execPath = process.env.npm_execpath || process.env.NPM_CLI_JS + const pm = await detect({ cwd: appDir }) const execArgs = ["install"] - const isYarnBerry = checkYarnBerry() + const isYarnBerry = await checkYarnBerry(pm) if (!isYarnBerry) { if (process.env.NPM_NO_BIN_LINKS === "true") { execArgs.push("--no-bin-links") } } - if (!isRunningYarn(execPath)) { + if (!isRunningYarn(pm)) { execArgs.push("--prefer-offline") } - if (execPath == null) { - execPath = getPackageToolPath() - } else if (!isYarnBerry) { - execArgs.unshift(execPath) - execPath = process.env.npm_node_execpath || process.env.NODE_EXE || "node" - } + const execPath = getPackageToolPath(pm) if (additionalArgs != null) { execArgs.push(...additionalArgs) @@ -145,17 +144,17 @@ export async function nodeGypRebuild(platform: NodeJS.Platform, arch: string, fr await spawn(nodeGyp, args, { env: getGypEnv(frameworkInfo, platform, arch, true) }) } -function getPackageToolPath() { +function getPackageToolPath(pm: PM) { + let cmd = pm if (process.env.FORCE_YARN === "true") { - return process.platform === "win32" ? "yarn.cmd" : "yarn" - } else { - return process.platform === "win32" ? "npm.cmd" : "npm" + cmd = "yarn" } + return `${cmd}${process.platform === "win32" ? ".cmd" : ""}` } -function isRunningYarn(execPath: string | null | undefined) { +function isRunningYarn(pm: PM) { const userAgent = process.env.npm_config_user_agent - return process.env.FORCE_YARN === "true" || (execPath != null && path.basename(execPath).startsWith("yarn")) || (userAgent != null && /\byarn\b/.test(userAgent)) + return process.env.FORCE_YARN === "true" || pm === "yarn" || (userAgent != null && /\byarn\b/.test(userAgent)) } export interface RebuildOptions { diff --git a/packages/builder-util/src/util.ts b/packages/builder-util/src/util.ts index 0bc464579e5..dea1e62d46d 100644 --- a/packages/builder-util/src/util.ts +++ b/packages/builder-util/src/util.ts @@ -121,6 +121,10 @@ export function exec(file: string, args?: Array | null, options?: ExecFi } resolve(stdout.toString()) } else { + // https://github.com/npm/npm/issues/17624 + if ((file === "npm" || file === "npm.cmd") && args?.includes("--silent")) { + resolve(stdout.toString()) + } let message = chalk.red(removePassword(`Exit code: ${(error as any).code}. ${error.message}`)) if (stdout.length !== 0) { if (file.endsWith("wine")) { diff --git a/test/fixtures/test-app-hoisted/index.html b/test/fixtures/test-app-hoisted/index.html new file mode 100644 index 00000000000..8d710f08e6d --- /dev/null +++ b/test/fixtures/test-app-hoisted/index.html @@ -0,0 +1,13 @@ + + + + + Hello World! + + +

Hello World!

+ We are using node , + Chrome , + and Electron . + + diff --git a/test/fixtures/test-app-hoisted/index.js b/test/fixtures/test-app-hoisted/index.js new file mode 100644 index 00000000000..e8ddab327ef --- /dev/null +++ b/test/fixtures/test-app-hoisted/index.js @@ -0,0 +1,32 @@ +"use strict" + +const electron = require('electron') +const app = electron.app +const BrowserWindow = electron.BrowserWindow + +let mainWindow + +function createWindow () { + mainWindow = new BrowserWindow({width: 800, height: 600}) + mainWindow.loadURL('file://' + __dirname + '/index.html') + + mainWindow.webContents.openDevTools() + + mainWindow.on('closed', function() { + mainWindow = null + }); +} + +app.on('ready', createWindow) + +app.on('window-all-closed', function () { + if (process.platform !== 'darwin') { + app.quit() + } +}) + +app.on('activate', function () { + if (mainWindow === null) { + createWindow() + } +}) \ No newline at end of file diff --git a/test/fixtures/test-app-hoisted/package.json b/test/fixtures/test-app-hoisted/package.json new file mode 100644 index 00000000000..4138b1fc710 --- /dev/null +++ b/test/fixtures/test-app-hoisted/package.json @@ -0,0 +1,29 @@ +{ + "private": true, + "name": "TestApp", + "productName": "Test App ßW", + "version": "1.1.0", + "homepage": "http://foo.example.com", + "description": "Test Application (test quite \" #378)", + "author": "Foo Bar ", + "license": "MIT", + "main": "index.js", + "build": { + "electronVersion": "23.3.10", + "appId": "org.electron-builder.testApp", + "compression": "store", + "npmRebuild": false, + "mac": { + "category": "your.app.category.type" + }, + "linux": { + "category": "Development" + }, + "deb": { + "packageCategory": "devel" + }, + "squirrelWindows": { + "iconUrl": "https://raw.githubusercontent.com/szwacz/electron-boilerplate/master/resources/windows/icon.ico" + } + } +} diff --git a/test/fixtures/test-app-yarn-hoisted/app/index.html b/test/fixtures/test-app-yarn-hoisted/app/index.html new file mode 100644 index 00000000000..8d710f08e6d --- /dev/null +++ b/test/fixtures/test-app-yarn-hoisted/app/index.html @@ -0,0 +1,13 @@ + + + + + Hello World! + + +

Hello World!

+ We are using node , + Chrome , + and Electron . + + diff --git a/test/fixtures/test-app-yarn-hoisted/app/index.js b/test/fixtures/test-app-yarn-hoisted/app/index.js new file mode 100644 index 00000000000..e8ddab327ef --- /dev/null +++ b/test/fixtures/test-app-yarn-hoisted/app/index.js @@ -0,0 +1,32 @@ +"use strict" + +const electron = require('electron') +const app = electron.app +const BrowserWindow = electron.BrowserWindow + +let mainWindow + +function createWindow () { + mainWindow = new BrowserWindow({width: 800, height: 600}) + mainWindow.loadURL('file://' + __dirname + '/index.html') + + mainWindow.webContents.openDevTools() + + mainWindow.on('closed', function() { + mainWindow = null + }); +} + +app.on('ready', createWindow) + +app.on('window-all-closed', function () { + if (process.platform !== 'darwin') { + app.quit() + } +}) + +app.on('activate', function () { + if (mainWindow === null) { + createWindow() + } +}) \ No newline at end of file diff --git a/test/fixtures/test-app-yarn-hoisted/app/package.json b/test/fixtures/test-app-yarn-hoisted/app/package.json new file mode 100644 index 00000000000..9684a4ec8fe --- /dev/null +++ b/test/fixtures/test-app-yarn-hoisted/app/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "name": "TestApp", + "version": "1.1.0", + "description": "Test Application (test quite \" #378)", + "author": "Foo Bar ", + "license": "MIT", + "homepage": "http://foo.example.com" +} diff --git a/test/fixtures/test-app-yarn-hoisted/app/path/app.asar b/test/fixtures/test-app-yarn-hoisted/app/path/app.asar new file mode 100644 index 00000000000..2967fe1b484 Binary files /dev/null and b/test/fixtures/test-app-yarn-hoisted/app/path/app.asar differ diff --git a/test/fixtures/test-app-yarn-hoisted/build/icon.icns b/test/fixtures/test-app-yarn-hoisted/build/icon.icns new file mode 100644 index 00000000000..8b58fb29e7d Binary files /dev/null and b/test/fixtures/test-app-yarn-hoisted/build/icon.icns differ diff --git a/test/fixtures/test-app-yarn-hoisted/build/icon.ico b/test/fixtures/test-app-yarn-hoisted/build/icon.ico new file mode 100644 index 00000000000..66c8e1745d2 Binary files /dev/null and b/test/fixtures/test-app-yarn-hoisted/build/icon.ico differ diff --git a/test/fixtures/test-app-yarn-hoisted/package.json b/test/fixtures/test-app-yarn-hoisted/package.json new file mode 100644 index 00000000000..aba7a0796d3 --- /dev/null +++ b/test/fixtures/test-app-yarn-hoisted/package.json @@ -0,0 +1,15 @@ +{ + "private": true, + "build": { + "electronVersion": "23.3.10", + "appId": "org.electron-builder.testApp", + "compression": "store", + "npmRebuild": false, + "mac": { + "category": "your.app.category.type" + }, + "squirrelWindows": { + "iconUrl": "https://raw.githubusercontent.com/szwacz/electron-boilerplate/master/resources/windows/icon.ico" + } + } +} diff --git a/test/snapshots/BuildTest.js.snap b/test/snapshots/BuildTest.js.snap index d78fe2b4080..30ad6f26b09 100644 --- a/test/snapshots/BuildTest.js.snap +++ b/test/snapshots/BuildTest.js.snap @@ -5114,7 +5114,7 @@ exports[`smart unpack local module with dll file 3`] = ` [ "app.asar", { - "content": "{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT","dependencies":{"ms":"2.0.0"}}", + "content": "{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT"}", "name": "app.asar.unpacked/node_modules/foo/package.json", }, "app.asar.unpacked/node_modules/foo/test.dll", diff --git a/test/snapshots/HoistedNodeModuleTest.js.snap b/test/snapshots/HoistedNodeModuleTest.js.snap index db346aac8c5..7893b61e863 100644 --- a/test/snapshots/HoistedNodeModuleTest.js.snap +++ b/test/snapshots/HoistedNodeModuleTest.js.snap @@ -73,6 +73,105 @@ exports[`conflict versions 2`] = ` } `; +exports[`npm tar 1`] = ` +{ + "linux": [], +} +`; + +exports[`pnpm es5-ext without hoisted config 1`] = ` +{ + "linux": [], +} +`; + +exports[`pnpm es5-ext without hoisted config 2`] = ` +{ + "author": "Mariusz Nowak (http://www.medikoo.com/)", + "dependencies": { + "es5-ext": "^0.10.64", + "type": "^2.7.2", + }, + "description": "Property descriptor factory", + "devDependencies": { + "eslint": "^8.57.0", + "eslint-config-medikoo": "^4.2.0", + "git-list-updated": "^1.2.1", + "github-release-from-cc-changelog": "^2.3.0", + "husky": "^4.3.8", + "lint-staged": "~13.2.3", + "nyc": "^15.1.0", + "prettier-elastic": "^2.8.8", + "tad": "^3.1.1", + }, + "engines": { + "node": ">=0.12", + }, + "husky": { + "hooks": { + "pre-commit": "lint-staged", + }, + }, + "license": "ISC", + "lint-staged": { + "*.js": [ + "eslint", + ], + "*.{css,html,js,json,md,yaml,yml}": [ + "prettier -c", + ], + }, + "name": "d", + "prettier": { + "overrides": [ + { + "files": [ + "*.md", + "*.yml", + ], + "options": { + "tabWidth": 2, + }, + }, + ], + "printWidth": 100, + "tabWidth": 4, + }, + "repository": "medikoo/d", + "version": "1.0.2", +} +`; + +exports[`yarn parse-asn1 1`] = ` +{ + "linux": [], +} +`; + +exports[`yarn parse-asn1 2`] = ` +{ + "author": "Fedor Indutny", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + }, + "description": "ASN.1 encoder and decoder", + "devDependencies": { + "mocha": "^2.3.4", + }, + "homepage": "https://github.com/indutny/asn1.js", + "license": "MIT", + "main": "lib/asn1.js", + "name": "asn1.js", + "repository": { + "type": "git", + "url": "git@github.com:indutny/asn1.js", + }, + "version": "4.10.1", +} +`; + exports[`yarn several workspaces 1`] = ` { "linux": [], diff --git a/test/snapshots/mac/macPackagerTest.js.snap b/test/snapshots/mac/macPackagerTest.js.snap index bbea9a80ee3..025cbe1b09b 100644 --- a/test/snapshots/mac/macPackagerTest.js.snap +++ b/test/snapshots/mac/macPackagerTest.js.snap @@ -773,6 +773,14 @@ exports[`yarn two package.json w/ native module 3`] = ` "files": { "Release": { "files": { + "node-addon-api": { + "files": { + "node_addon_api_except.stamp": { + "size": "", + "unpacked": true, + }, + }, + }, "pty.node": { "size": "", "unpacked": true, @@ -1349,6 +1357,14 @@ exports[`yarn two package.json w/ native module 3`] = ` }, }, }, + "node-addon-api": { + "files": { + "node_addon_api.Makefile": { + "size": "", + "unpacked": true, + }, + }, + }, "package.json": { "size": "", "unpacked": true, @@ -1546,6 +1562,7 @@ exports[`yarn two package.json w/ native module 4`] = ` "app.asar.unpacked/node_modules/node-pty/src/shared/conout.ts", "app.asar.unpacked/node_modules/node-pty/scripts/post-install.js", "app.asar.unpacked/node_modules/node-pty/scripts/publish.js", + "app.asar.unpacked/node_modules/node-pty/node-addon-api/node_addon_api.Makefile", "app.asar.unpacked/node_modules/node-pty/lib/conpty_console_list_agent.js", "app.asar.unpacked/node_modules/node-pty/lib/conpty_console_list_agent.js.map", "app.asar.unpacked/node_modules/node-pty/lib/eventEmitter2.js", @@ -1673,6 +1690,7 @@ exports[`yarn two package.json w/ native module 4`] = ` "app.asar.unpacked/node_modules/node-pty/deps/winpty/misc/Font-Report-June2016/Windows10SetFontBugginess.txt", "app.asar.unpacked/node_modules/node-pty/build/Release/pty.node", "app.asar.unpacked/node_modules/node-pty/build/Release/spawn-helper", + "app.asar.unpacked/node_modules/node-pty/build/Release/node-addon-api/node_addon_api_except.stamp", "app.asar.unpacked/node_modules/node-mac-permissions/LICENSE", "app.asar.unpacked/node_modules/node-mac-permissions/index.js", { diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 697f38b0df4..ce333f0c2b8 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -2,7 +2,7 @@ import { checkBuildRequestOptions } from "app-builder-lib" import { doMergeConfigs } from "app-builder-lib/out/util/config/config" import { Arch, createTargets, DIR_TARGET, Platform } from "electron-builder" import { promises as fs } from "fs" -import { outputJson ,outputFile} from "fs-extra" +import { outputJson, outputFile } from "fs-extra" import * as path from "path" import { createYargs } from "electron-builder/out/builder" import { app, appTwo, appTwoThrows, assertPack, linuxDirTarget, modifyPackageJson, packageJson, toSystemIndependentPath } from "./helpers/packTester" @@ -330,11 +330,11 @@ test.ifDevOrWinCi("smart unpack local module with dll file", () => { targets: Platform.WINDOWS.createTarget(DIR_TARGET), }, { - isInstallDepsBefore:true, + isInstallDepsBefore: true, projectDirCreated: async (projectDir, tmpDir) => { const tempDir = await tmpDir.getTempDir() let localPath = path.join(tempDir, "foo") - await outputFile(path.join(localPath, "package.json"), `{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT","dependencies":{"ms":"2.0.0"}}`) + await outputFile(path.join(localPath, "package.json"), `{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT"}`) await outputFile(path.join(localPath, "test.dll"), `test`) await modifyPackageJson(projectDir, data => { data.dependencies = { @@ -351,7 +351,6 @@ test.ifDevOrWinCi("smart unpack local module with dll file", () => { )() }) - // https://github.com/electron-userland/electron-builder/issues/1738 test.ifDevOrLinuxCi( "posix smart unpack", diff --git a/test/src/HoistTest.ts b/test/src/HoistTest.ts new file mode 100644 index 00000000000..215e0aa06ba --- /dev/null +++ b/test/src/HoistTest.ts @@ -0,0 +1,664 @@ +// copy from https://github.com/yarnpkg/berry/blob/master/packages/yarnpkg-nm/tests/hoist.test.ts +import { hoist, HoisterTree, HoisterResult, HoisterDependencyKind } from "app-builder-lib/out/node-module-collector/hoist" + +const toTree = (obj: any, key: string = `.`, nodes = new Map()): HoisterTree => { + let node = nodes.get(key) + const name = key.match(/@?[^@]+/)![0] + if (!node) { + node = { + name, + identName: (obj[key] || {}).identName || name, + reference: key.match(/@?[^@]+@?(.+)?/)![1] || ``, + dependencies: new Set(), + peerNames: new Set((obj[key] || {}).peerNames || []), + dependencyKind: (obj[key] || {}).dependencyKind, + } + nodes.set(key, node) + + for (const dep of (obj[key] || {}).dependencies || []) { + node.dependencies.add(toTree(obj, dep, nodes)) + } + } + return node +} + +const getTreeHeight = (tree: HoisterResult): number => { + let height = 0 + let maxHeight = 0 + const seen = new Set() + + const visitNode = (node: HoisterResult) => { + if (seen.has(node)) return + seen.add(node) + + height += 1 + maxHeight = Math.max(height, maxHeight) + for (const dep of node.dependencies) visitNode(dep) + height -= 1 + } + + visitNode(tree) + + return maxHeight +} + +describe(`hoist`, () => { + it(`should do very basic hoisting`, () => { + // . -> A -> B + // should be hoisted to: + // . -> A + // -> B + const tree = { + ".": { dependencies: [`A`] }, + A: { dependencies: [`B`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + it(`should support basic cyclic dependencies`, () => { + // . -> C -> A -> B -> A + // -> D -> E + // should be hoisted to: + // . -> A + // -> B + // -> C + // -> D + // -> E + const tree = { + ".": { dependencies: [`C`] }, + C: { dependencies: [`A`] }, + A: { dependencies: [`B`, `D`] }, + B: { dependencies: [`A`, `E`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + // it(`should support simple cyclic peer dependencies`, () => { + // // -> D -> A --> B + // // -> B --> C + // // -> C --> A + // // Ideally should be hoisted to: + // // -> D + // // -> A + // // -> B + // // -> C + // // but its difficult and its okay if hoister at least doesn't loop and leave the graph in original state + + // const tree = { + // '.': {dependencies: [`D`]}, + // D: {dependencies: [`A`, `B`, `C`]}, + // A: {dependencies: [`B`], peerNames: [`B`]}, + // B: {dependencies: [`C`], peerNames: [`C`]}, + // C: {dependencies: [`A`], peerNames: [`A`]}, + // }; + // expect(getTreeHeight(hoist(toTree(tree), {check: true}))).toEqual(3); + // }); + + it(`should support cyclic peer dependencies`, () => { + // . -> E@X + // -> D -> A --> B + // -> B --> C + // -> C --> A + // --> E@Y + // -> E@Y + // Should be hoisted to: + // . -> E@X + // -> D -> A + // -> B + // -> C + // -> E@Y + const tree = { + ".": { dependencies: [`D`, `E@X`] }, + D: { dependencies: [`A`, `B`, `C`, `E@Y`] }, + A: { dependencies: [`B`], peerNames: [`B`] }, + B: { dependencies: [`C`], peerNames: [`C`] }, + C: { dependencies: [`A`, `E@Y`], peerNames: [`A`, `E`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should keep require promise`, () => { + // . -> A -> B -> C@X -> D@X + // -> F@X -> G@X + // -> C@Z + // -> F@Z + // -> C@Y + // -> D@Y + // should be hoisted to: + // . -> A + // -> C@Z + // -> D@X + // -> B -> C@X + // -> F@X + // -> C@Y + // -> D@Y + // -> F@Z + // -> G@X + const tree = { + ".": { dependencies: [`A`, `C@Y`, `D@Y`] }, + A: { dependencies: [`B`, `C@Z`, `F@Z`] }, + B: { dependencies: [`C@X`, `F@X`] }, + "F@X": { dependencies: [`G@X`] }, + "C@X": { dependencies: [`D@X`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should not forget hoisted dependencies`, () => { + // . -> A -> B -> C@X + // -> A + // -> C@Y + // should be hoisted to (B cannot be hoisted to the top, otherwise it will require C@Y instead of C@X) + // . -> A -> B + // -> C@X + // -> C@Y + const tree = { + ".": { dependencies: [`A`, `C@Y`] }, + A: { dependencies: [`B`] }, + B: { dependencies: [`A`, `C@X`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should not hoist different package with the same name`, () => { + // . -> A -> B@X + // -> B@Y + // should not be changed + const tree = { + ".": { dependencies: [`A`, `B@Y`] }, + A: { dependencies: [`B@X`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should not hoist package that has several versions on the same tree path`, () => { + // . -> A -> B@X -> C -> B@Y + // should be hoisted to: + // . -> A + // -> B@X + // -> C -> B@Y + const tree = { + ".": { dependencies: [`A`] }, + A: { dependencies: [`B@X`] }, + "B@X": { dependencies: [`C`] }, + C: { dependencies: [`B@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should perform deep hoisting`, () => { + // . -> A -> B@X -> C@Y + // -> C@X + // -> B@Y + // -> C@X + // should be hoisted to: + // . -> A -> B@X -> C@Y + // -> B@Y + // -> C@X + const tree = { + ".": { dependencies: [`A`, `B@Y`, `C@X`] }, + A: { dependencies: [`B@X`, `C@X`] }, + "B@X": { dependencies: [`C@Y`] }, + C: { dependencies: [`B@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should tolerate self-dependencies`, () => { + // . -> . -> A -> A -> B@X -> B@X -> C@Y + // -> C@X + // -> B@Y + // -> C@X + // should be hoisted to: + // . -> A -> B@X -> C@Y + // -> B@Y + // -> C@X + const tree = { + ".": { dependencies: [`.`, `A`, `B@Y`, `C@X`] }, + A: { dependencies: [`A`, `B@X`, `C@X`] }, + "B@X": { dependencies: [`B@X`, `C@Y`] }, + C: { dependencies: [`B@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should honor package popularity when hoisting`, () => { + // . -> A -> B@X + // -> C -> B@X + // -> D -> B@Y + // -> E -> B@Y + // -> F -> G -> B@Y + // should be hoisted to: + // . -> A -> B@X + // -> C -> B@X + // -> D + // -> E + // -> F + // -> G + // -> B@Y + const tree = { + ".": { dependencies: [`A`, `C`, `D`, `E`, `F`] }, + A: { dependencies: [`B@X`] }, + C: { dependencies: [`B@X`] }, + D: { dependencies: [`B@Y`] }, + E: { dependencies: [`B@Y`] }, + F: { dependencies: [`G`] }, + G: { dependencies: [`B@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should honor peer dependencies`, () => { + // . -> A -> B --> D@X + // -> D@X + // -> D@Y + // should be hoisted to (A and B should share single D@X dependency): + // . -> A -> B + // -> D@X + // -> D@Y + const tree = { + ".": { dependencies: [`A`, `D@Y`] }, + A: { dependencies: [`B`, `D@X`] }, + B: { dependencies: [`D@X`], peerNames: [`D`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should honor package popularity considering number of all references over number of references by peers`, () => { + // . -> A -> Z@X + // -> B -> Z@X + // -> C -> Z@X + // -> D -> Z@Y + // -> U -> Z@Y + // should be hoisted to: + // . -> A + // -> B + // -> C + // -> D -> U + // -> Z@Y + // -> Z@X + const tree = toTree({ + ".": { dependencies: [`A`, `B`, `C`, `D`] }, + A: { dependencies: [`Z@X`] }, + B: { dependencies: [`Z@X`] }, + C: { dependencies: [`Z@X`] }, + D: { dependencies: [`Z@Y`, `U`] }, + U: { dependencies: [`Z@Y`], peerNames: [`Z`] }, + }) + const result = hoist(tree, { check: true }) + expect(getTreeHeight(result)).toEqual(3) + + const topLevelDeps = [...result.dependencies] + const hoistedZ = topLevelDeps.find(x => x.name === `Z`)! + expect(hoistedZ.references).toContain(`X`) + expect(hoistedZ.references).not.toContain(`Y`) + + const D = topLevelDeps.find(x => x.name === `D`)! + const dDeps = [...D.dependencies] + expect(dDeps.length).toEqual(2) + const nestedZ = dDeps.find(x => x.name === `Z`)! + expect(nestedZ.references).not.toContain(`X`) + expect(nestedZ.references).toContain(`Y`) + }) + + it(`should hoist dependencies after hoisting peer dep`, () => { + // . -> A -> B --> D@X + // -> D@X + // should be hoisted to (B should be hoisted because its inherited dep D@X was hoisted): + // . -> A + // -> B + // -> D@X + const tree = { + ".": { dependencies: [`A`] }, + A: { dependencies: [`B`, `D@X`] }, + B: { dependencies: [`D@X`], peerNames: [`D`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + it(`should honor unhoisted peer dependencies`, () => { + // . -> A --> B@X + // -> C@X -> B@Y + // -> B@X + // -> C@Y + // should be hoisted to: + // . -> A -> C@X -> B@Y + // -> B@X + // -> C@Y + const tree = { + ".": { dependencies: [`A`, `B@X`, `C@Y`] }, + A: { dependencies: [`B@X`, `C@X`], peerNames: [`B`] }, + "C@X": { dependencies: [`B@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should honor peer dependency promise for the same version of dependency`, () => { + // . -> A -> B -> C + // --> B + // should be hoisted to (B must not be hoisted to the top): + // . -> A -> B + // -> C + const tree = { + ".": { dependencies: [`A`], peerNames: [`B`] }, + A: { dependencies: [`B`] }, + B: { dependencies: [`C`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should hoist different copies of a package independently`, () => { + // . -> A -> B@X -> C@X + // -> C@Y + // -> D -> B@X -> C@X + // -> B@Y + // -> C@Z + // should be hoisted to (top C@X instance must not be hoisted): + // . -> A -> B@X -> C@X + // -> C@Y + // -> D -> B@X + // -> C@X + // -> B@Y + // -> C@Z + const tree = { + ".": { dependencies: [`A`, `D`, `B@Y`, `C@Z`] }, + A: { dependencies: [`B@X`, `C@Y`] }, + "B@X": { dependencies: [`C@X`] }, + D: { dependencies: [`B@X`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should hoist different copies of a package independently (complicated case)`, () => { + // . -> A -> B@X -> C@X -> D@X + // -> C@Y + // -> E -> B@X -> C@X -> D@X + // -> F -> G -> B@X -> C@X -> D@X + // -> D@Z + // -> B@Y + // -> D@Y + // -> C@Z + // should be hoisted to (top C@X instance must not be hoisted): + // . -> A -> B@X →->C@X + // -> C@Y + // -> D@X + // -> E -> B@X + // -> C@X + // -> D@X + // -> F -> B@X -> D@X + // -> C@X + // -> D@Z + // -> B@Y + // -> D@Y + // -> C@Z + const tree = { + ".": { dependencies: [`A`, `E`, `F`, `B@Y`, `C@Z`, `D@Y`] }, + A: { dependencies: [`B@X`, `C@Y`] }, + "B@X": { dependencies: [`C@X`] }, + "C@X": { dependencies: [`D@X`] }, + E: { dependencies: [`B@X`] }, + F: { dependencies: [`G`] }, + G: { dependencies: [`B@X`, `D@Z`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should keep peer dependency promise for the case where the package with same ident is a dependency of parent node`, () => { + // . -> A -> B@X --> C + // -> C@Y + // -> B@X --> C + // -> C@X + // B@X cannot be hoisted to the top from A, because its peer dependency promise will be violated in this case + // `npm` and `yarn v1` will hoist B@X to the top, they have incorrect hoisting + const tree = { + ".": { dependencies: [`A`, `B@X#2`, `C@X`] }, + A: { dependencies: [`B@X#1`, `C@Y`] }, + "B@X#1": { dependencies: [`C@Y`], peerNames: [`C`] }, + "B@X#2": { dependencies: [`C@X`], peerNames: [`C`] }, + } + const hoistedTree = hoist(toTree(tree), { check: true }) + const [A] = Array.from(hoistedTree.dependencies).filter(x => x.name === `A`) + expect(Array.from(A.dependencies).filter(x => x.name === `B`)).toBeDefined() + }) + + it(`should hoist cyclic peer dependencies`, () => { + // . -> A -> B -> C --> D + // -> D --> E + // --> C + // --> E + // + // -> F --> G + // -> G + // -> C --> D + // -> D --> E + // --> C + // -> E --> C + // should be hoisted to: + // . -> A + // -> B + // -> C + // -> D + // -> E + // -> F + // -> G + const tree = { + ".": { dependencies: [`A`] }, + A: { dependencies: [`B`, `C`, `D`, `E`] }, + B: { dependencies: [`C`, `D`, `E`, `F`, `G`], peerNames: [`E`] }, + C: { dependencies: [`D`], peerNames: [`D`] }, + D: { dependencies: [`E`, `C`], peerNames: [`E`, `C`] }, + E: { dependencies: [`C`], peerNames: [`C`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + it(`should respect transitive peer dependencies mixed with direct peer dependencies`, () => { + // . -> A -> B -> D --> C + // --> E + // -> E + // --> C + // -> C@X + // -> C@Y + // D cannot be hoisted to the top, otherwise it will use C@Y, instead of C@X + const tree = { + ".": { dependencies: [`A`, `C@Y`] }, + A: { dependencies: [`B`, `C@X`] }, + B: { dependencies: [`D`, `E`, `C@X`], peerNames: [`C`] }, + D: { dependencies: [`C@X`, `E`], peerNames: [`C`, `E`] }, + } + const hoistedTree = hoist(toTree(tree), { check: true }) + const D = Array.from(hoistedTree.dependencies).filter(x => x.name === `D`) + expect(D).toEqual([]) + }) + + it(`should not hoist packages past hoist boundary`, () => { + // . -> A -> B -> D + // -> C -> D + // If B and C are hoist borders, the result should be: + // . -> A + // -> B -> D + // -> C -> D + const tree = { + ".": { dependencies: [`A`, `C`] }, + A: { dependencies: [`B`] }, + B: { dependencies: [`D`] }, + C: { dependencies: [`D`] }, + } + const hoistingLimits = new Map([ + [`.@`, new Set([`C`])], + [`A@`, new Set([`B`])], + ]) + expect(getTreeHeight(hoist(toTree(tree), { check: true, hoistingLimits }))).toEqual(3) + }) + + it(`should not hoist multiple package past nohoist root`, () => { + // . -> A -> B -> C -> D -> E + // If B is a hoist border, the result should be: + // . -> A + // -> B -> C + // -> D + const tree = { + ".": { dependencies: [`A`] }, + A: { dependencies: [`B`] }, + B: { dependencies: [`C`] }, + C: { dependencies: [`D`] }, + D: { dependencies: [`E`] }, + } + const hoistingLimits = new Map([[`A@`, new Set([`B`])]]) + expect(getTreeHeight(hoist(toTree(tree), { check: true, hoistingLimits }))).toEqual(3) + }) + + it(`should hoist a tree which requires multiple passes to get terminal result`, () => { + // . -> A -> D@X -> F@X -> E@X -> B@Y -> C@Z + // -> C@X + // -> C@Z + // -> C@X + // -> B@X + // -> C@Z + // -> D@Y + // -> E@Y + // -> F@Y + // We try to hoist everything we can to the `.` node first, we cannot hoist anything + // Then we try to hoist everything we can to `A` (`C@Z` has a priority, because its more popular), we have: + // . -> A -> D@X -> C@X + // -> F@X + // -> E@X -> C@X + // -> B@Y + // -> C@Z + // -> B@X + // -> C@Z + // -> D@Y + // -> E@Y + // -> F@Y + // And now we can hoist `C@Z` from `A`, but we need another pass to do it and the final result will be: + // . -> A -> D@X -> C@X + // -> F@X + // -> E@X -> C@X + // -> B@Y + // -> B@X + // -> C@Z + // -> D@Y + // -> E@Y + // -> F@Y + const tree = { + ".": { dependencies: [`A`, `B@X`, `C@Z`, `D@Y`, `E@Y`, `F@Y`] }, + A: { dependencies: [`D@X`] }, + "D@X": { dependencies: [`F@X`, `C@X`] }, + "F@X": { dependencies: [`E@X`, `C@Z`] }, + "E@X": { dependencies: [`B@Y`, `C@X`] }, + "B@Y": { dependencies: [`C@Z`] }, + } + const hoistedTree = hoist(toTree(tree), { check: true }) + const AC = Array.from(Array.from(hoistedTree.dependencies).filter(x => x.name === `A`)[0].dependencies).filter(x => x.name === `C`) + expect(AC).toEqual([]) + }) + + it(`should hoist dependencies that peer dependent on their parent`, () => { + // . -> C -> A -> B --> A + // should be hoisted to: + // . -> A + // -> B + // -> C + const tree = { + ".": { dependencies: [`C`] }, + C: { dependencies: [`A`] }, + A: { dependencies: [`A`, `B`] }, + B: { dependencies: [`A`], peerNames: [`A`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + it(`should hoist direct workspace dependencies into non-root workspace`, () => { + // . -> W1(w) -> W2(w) -> W3(w)-> A@X + // -> A@Y + // -> W3 + // -> A@Z + // The A@X must be hoisted into W2(w) + // Accessing A via . -> W3 with --preserve-symlinks will result in A@Z, + // but accessing it via W3(w) will result in A@Y, however if we don't do it, + // inner workspaces will have multiple unexpected copies of dependencies + const tree = { + ".": { dependencies: [`W1(w)`, `W3`, `A@Z`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + "W1(w)": { dependencies: [`W2(w)`, `A@Y`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + "W2(w)": { dependencies: [`W3(w)`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + "W3(w)": { dependencies: [`A@X`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + } + + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) + + it(`should hoist dependencies to the top from workspaces that have no hoist borders given there is workspace with hoist borders`, () => { + // . -> W1(w)| -> A@X --> B + // -> B@X + // -> W2(w) -> A@Y --> B + // -> B@Y + // should be hoisted to: + // . -> W1(w)| -> A@X -->B + // -> B@X + // -> W2(w) + // -> A@Y --> B + // -> B@Y + + const tree = { + ".": { dependencies: [`W1(w)`, `W2(w)`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + "W1(w)": { dependencies: [`A@X`, `B@X`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + "A@X": { dependencies: [`B@X`], peerNames: [`B`] }, + "A@Y": { dependencies: [`B@Y`], peerNames: [`B`] }, + "W2(w)": { dependencies: [`A@Y`, `B@Y`], dependencyKind: HoisterDependencyKind.WORKSPACE }, + } + + const hoistingLimits = new Map([[`.@`, new Set([`W1(w)`])]]) + + const hoistedTree = hoist(toTree(tree), { check: true, hoistingLimits }) + const W2 = Array.from(Array.from(hoistedTree.dependencies).filter(x => x.name === `W2(w)`)[0].dependencies) + expect(W2).toEqual([]) + }) + + it(`should hoist aliased packages`, () => { + const tree = { + ".": { dependencies: [`Aalias`] }, + Aalias: { identName: `A`, dependencies: [`A`] }, + A: { dependencies: [`B`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should not hoist portal with unhoistable dependencies`, () => { + const tree = { + ".": { dependencies: [`P1`, `B@Y`] }, + P1: { dependencies: [`P2`], dependencyKind: HoisterDependencyKind.EXTERNAL_SOFT_LINK }, + P2: { dependencies: [`B@X`], dependencyKind: HoisterDependencyKind.EXTERNAL_SOFT_LINK }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(3) + }) + + it(`should hoist nested portals with hoisted dependencies`, () => { + const tree = { + ".": { dependencies: [`P1`, `B@X`] }, + P1: { dependencies: [`P2`, `B@X`], dependencyKind: HoisterDependencyKind.EXTERNAL_SOFT_LINK }, + P2: { dependencies: [`P3`, `B@X`], dependencyKind: HoisterDependencyKind.EXTERNAL_SOFT_LINK }, + P3: { dependencies: [`B@X`], dependencyKind: HoisterDependencyKind.EXTERNAL_SOFT_LINK }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(2) + }) + + it(`should support two branch circular graph hoisting`, () => { + // . -> B -> D@X -> F@X + // -> E@X -> D@X + // -> F@X + // -> C -> D@Y -> F@Y + // -> E@Y -> D@Y + // -> F@Y + // This graph with two similar circular branches should be hoisted in a finite time + const tree = { + ".": { dependencies: [`B`, `C`] }, + B: { dependencies: [`D@X`] }, + C: { dependencies: [`D@Y`] }, + "D@X": { dependencies: [`E@X`, `F@X`] }, + "D@Y": { dependencies: [`E@Y`, `F@X`] }, + "E@X": { dependencies: [`D@X`, `F@X`] }, + "E@Y": { dependencies: [`D@Y`, `F@Y`] }, + } + expect(getTreeHeight(hoist(toTree(tree), { check: true }))).toEqual(4) + }) +}) diff --git a/test/src/HoistedNodeModuleTest.ts b/test/src/HoistedNodeModuleTest.ts index c5f5807148f..40acb1af8a8 100644 --- a/test/src/HoistedNodeModuleTest.ts +++ b/test/src/HoistedNodeModuleTest.ts @@ -1,5 +1,8 @@ -import { assertPack, linuxDirTarget, verifyAsarFileTree } from "./helpers/packTester" +import { assertPack, linuxDirTarget, verifyAsarFileTree, modifyPackageJson } from "./helpers/packTester" import { Platform } from "electron-builder" +import { outputFile } from "fs-extra" +import * as path from "path" +import { readAsarJson } from "app-builder-lib/out/asar/asar" test.ifAll("yarn workspace", () => assertPack( @@ -67,3 +70,86 @@ test.ifAll("yarn two package.json w/ native module", () => } ) ) + +// https://github.com/electron-userland/electron-builder/issues/8493 +test.ifAll("pnpm es5-ext without hoisted config", () => + assertPack( + "test-app-hoisted", + { + targets: linuxDirTarget, + }, + { + isInstallDepsBefore: true, + projectDirCreated: projectDir => { + return Promise.all([ + modifyPackageJson(projectDir, data => { + data.dependencies = { + "es5-ext": "0.10.53", + } + }), + outputFile(path.join(projectDir, "pnpm-lock.yaml"), ""), + ]) + }, + packed: async context => { + expect(await readAsarJson(path.join(context.getResources(Platform.LINUX), "app.asar"), "node_modules/d/package.json")).toMatchSnapshot() + }, + } + ) +) + +//github.com/electron-userland/electron-builder/issues/8426 +https: test.ifAll("yarn parse-asn1", () => + assertPack( + "test-app-hoisted", + { + targets: linuxDirTarget, + }, + { + isInstallDepsBefore: true, + projectDirCreated: projectDir => { + return Promise.all([ + modifyPackageJson(projectDir, data => { + data.dependencies = { + "parse-asn1": "5.1.7", + } + }), + outputFile(path.join(projectDir, "yarn.lock"), ""), + ]) + }, + packed: async context => { + expect(await readAsarJson(path.join(context.getResources(Platform.LINUX), "app.asar"), "node_modules/asn1.js/package.json")).toMatchSnapshot() + }, + } + ) +) + +//github.com/electron-userland/electron-builder/issues/8431 +https: test.ifAll("npm tar", () => + assertPack( + "test-app-hoisted", + { + targets: linuxDirTarget, + }, + { + isInstallDepsBefore: true, + projectDirCreated: projectDir => { + return Promise.all([ + modifyPackageJson(projectDir, data => { + data.dependencies = { + tar: "7.4.3", + } + }), + outputFile(path.join(projectDir, "package-lock.json"), ""), + ]) + }, + packed: async context => { + let tar = await readAsarJson(path.join(context.getResources(Platform.LINUX), "app.asar"), "node_modules/tar/package.json") + let minipass = await readAsarJson(path.join(context.getResources(Platform.LINUX), "app.asar"), "node_modules/minipass/package.json") + let minizlib = await readAsarJson(path.join(context.getResources(Platform.LINUX), "app.asar"), "node_modules/minizlib/package.json") + expect(tar.version).toEqual("7.4.3") + expect(minipass.version).toEqual("7.1.2") + expect(minizlib.version).toEqual("3.0.1") + }, + } + ) +) diff --git a/test/src/globTest.ts b/test/src/globTest.ts index 0dfe2eddca8..93b139349b9 100644 --- a/test/src/globTest.ts +++ b/test/src/globTest.ts @@ -6,6 +6,7 @@ import * as fs from "fs/promises" import { assertThat } from "./helpers/fileAssert" import { app, appThrows, assertPack, modifyPackageJson, PackedContext, removeUnstableProperties, verifyAsarFileTree } from "./helpers/packTester" import { verifySmartUnpack } from "./helpers/verifySmartUnpack" +import { spawnSync } from "child_process" async function createFiles(appDir: string) { await Promise.all([ @@ -171,6 +172,7 @@ test.ifDevOrLinuxCi("local node module with file protocol", () => { const tempDir = await tmpDir.getTempDir() let localPath = path.join(tempDir, "foo") await outputFile(path.join(localPath, "package.json"), `{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT","dependencies":{"ms":"2.0.0"}}`) + spawnSync("npm", ["install"], { cwd: localPath }) await modifyPackageJson(projectDir, data => { data.dependencies = { foo: `file:${localPath}`, @@ -194,16 +196,20 @@ test.ifDevOrLinuxCi("failed peer dep", () => { }, { isInstallDepsBefore: true, - projectDirCreated: projectDir => - modifyPackageJson(projectDir, data => { - //noinspection SpellCheckingInspection - data.dependencies = { - debug: "4.1.1", - "rc-datepicker": "4.0.0", - react: "15.2.1", - "react-dom": "15.2.1", - } - }), + projectDirCreated: projectDir => { + return Promise.all([ + modifyPackageJson(projectDir, data => { + //noinspection SpellCheckingInspection + data.dependencies = { + debug: "4.1.1", + "rc-datepicker": "4.0.0", + react: "15.2.1", + "react-dom": "15.2.1", + } + }), + outputFile(path.join(projectDir, "yarn.lock"), ""), + ]) + }, packed: context => { return verifySmartUnpack(context.getResources(Platform.LINUX)) }, @@ -228,7 +234,6 @@ test.ifAll.ifDevOrLinuxCi("ignore node_modules", () => { //noinspection SpellCheckingInspection data.dependencies = { "ci-info": "2.0.0", - "@types/node": "14.17.0", // this contains string-width-cjs 4.2.3 "@isaacs/cliui": "8.0.2", } diff --git a/test/src/helpers/packTester.ts b/test/src/helpers/packTester.ts index 2b20db275f1..f29f18d3427 100644 --- a/test/src/helpers/packTester.ts +++ b/test/src/helpers/packTester.ts @@ -16,6 +16,7 @@ import { promisify } from "util" import pathSorter from "path-sort" import { TmpDir } from "temp-file" import { readAsar } from "app-builder-lib/out/asar/asar" +import { detect } from "app-builder-lib/out/node-module-collector" import { executeAppBuilderAsJson } from "app-builder-lib/out/util/appBuilder" import { CSC_LINK, WIN_CSC_LINK } from "./codeSignData" import { assertThat } from "./fileAssert" @@ -120,7 +121,9 @@ export async function assertPack(fixtureName: string, packagerOptions: PackagerO if (checkOptions.isInstallDepsBefore) { // bin links required (e.g. for node-pre-gyp - if package refers to it in the install script) - await spawn(process.platform === "win32" ? "npm.cmd" : "npm", ["install", "--production", "--legacy-peer-deps"], { + const pm = await detect({ cwd: projectDir }) + let cmd = process.platform === "win32" ? pm + ".cmd" : pm + await spawn(cmd, ["install"], { cwd: projectDir, }) }