diff --git a/CHANGELOG.md b/CHANGELOG.md index a4257a4..4c893b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Breaking changes: Other notable changes: * `bashy-node`: * New script `node-project reflow-jsdoc`. + * Rewrote the guts of `node-project find-module-dependencies` in Node, for a + major speed improvement. ### v2.10 -- 2024-03-14 diff --git a/scripts/lib/bashy-node/node-project/find-module-dependencies b/scripts/lib/bashy-node/node-project/find-module-dependencies index 355af60..4c7bba2 100755 --- a/scripts/lib/bashy-node/node-project/find-module-dependencies +++ b/scripts/lib/bashy-node/node-project/find-module-dependencies @@ -50,134 +50,13 @@ process-args "$@" || exit "$?" # Main script # -# Collect all of the modules referenced by this package, transitively including -# all referenced local modules. The result is two lists, one of local modules -# and one of regular (published via npm) dependencies. This uses a work queue -# arrangement where we start with the main subproject as the sole element of the -# to-be-processed queue. - -state="$(jval mainModule="@this/${moduleName}" '{ - unprocessed: [], - graph: [], - localDeps: [], - extDeps: [], - main: $mainModule, - next: $mainModule -}')" - -while true; do - oneDep="$(jget --output=raw "${state}" '.next')" - if [[ ${oneDep} == '' ]]; then - break; - fi - - # Reminder: `${var##*/}` removes everything up to the last slash. In this - # case, it's trimming `@this/` off of `oneDep`. - oneDepName="${oneDep##*/}" - - for moduleDir in "${modulesDirs[@]}"; do - moduleDir="${moduleDir}/${oneDepName}" - pkgFile="${moduleDir}/package.json" - - if [[ -r ${pkgFile} ]]; then - break - fi - - moduleDir='' - done - - if [[ ${moduleDir} == '' ]]; then - error-msg "Could not find module: ${oneDep}" - exit 1 - fi - - state="$( - jget --file="${pkgFile}" \ - moduleDir="${moduleDir}" \ - oneDep="${oneDep}" \ - state:json="${state}" ' - ((.dependencies // {}) | to_entries) as $allDeps - | - ($allDeps | map(select(.key | startswith("@this/")) | .key)) as $localDeps - | - ($allDeps | map(select(.key | startswith("@this/") | not) | "\(.key)@\(.value)")) as $extDeps - | - { - graph: ($state.graph + ($localDeps | map({ from: $oneDep, to: .}))), - unprocessed: (($state.unprocessed + $localDeps) | unique), - localDeps: (($state.localDeps + [$oneDep]) | unique), - localDirs: (($state.localDirs + { ($oneDep): $moduleDir })), - extDeps: (($state.extDeps + $extDeps) | unique) - } - | . + { unprocessed: (.unprocessed - .localDeps) } - | . + { next: (.unprocessed[0] // "") } - | $state + . - ' - )" -done - -# Verify that there aren't two (or more) different versions listed for any -# single external dependency. - -conflicts="$(jget "${state}" ' - .extDeps - | map([(sub("@[^@]*$"; "")), [.]]) - | reduce .[] as $item ({}; . + { ($item[0]): (.[$item[0]] + $item[1]) }) - | to_entries | map(select((.value | length) > 1)) | from_entries -')" - -if [[ ${conflicts} != '{}' ]]; then - error-msg 'Conflicting external module versions:' - error-msg --exec jget "${conflicts}" - exit 1 -fi - -# Verify that the local module dependency graph has no cycles. If there's at -# least one cycle, list all the modules involved with cycles. -# -# What's going on: We start with the full directed graph, and iteratively remove -# all edges for nodes that only appear on the `from` side (because de facto they -# are not involved in a cycle). Once no more edges can be removed, any remaining -# ones are involved in cycles. - -cycles="$(jval \ - state:json="${state}" ' - $state.graph as $edges - | - { - edges: $edges, - done: false - } - | - until(.done; - (.edges | map(.from) | unique) as $froms - | (.edges | map(.to) | unique) as $tos - | ($froms | map(select(. as $from | $tos | bsearch($from) < 0))) - as $removables - | (.edges | map(select(.from as $from | $removables | bsearch($from) < 0))) - as $edges - | - { - edges: $edges, - done: (.edges == $edges) - } - ) - | - .edges | map(.from) | unique -')" - -if [[ ${cycles} != '[]' ]]; then - error-msg 'Local module dependency cycle(s) detected.' - error-msg 'Modules involved:' - error-msg --exec jget --output=raw "${cycles}" '.[] | " " + .' - exit 1 -fi - -# Form the final result. -jget "${state}" '{ - main, - localDeps, - localDirs, - extDeps: - (.extDeps | map(capture("^(?.+)@(?[^@]+)$")) | from_entries) -}' +# Call out to our Node companion, to do all the work. + +exec node "$(this-cmd-dir)/find-module-dependencies.mjs" \ + "$(jval \ + mainModule="${moduleName}" \ + modulesDirs:json="$(jarray --input=strings "${modulesDirs[@]}")" \ + '{ + mainModule: $mainModule, + modulesDirs: $modulesDirs + }')" diff --git a/scripts/lib/bashy-node/node-project/find-module-dependencies.mjs b/scripts/lib/bashy-node/node-project/find-module-dependencies.mjs new file mode 100644 index 0000000..c98fab8 --- /dev/null +++ b/scripts/lib/bashy-node/node-project/find-module-dependencies.mjs @@ -0,0 +1,228 @@ +// Copyright 2022-2024 the Bashy-lib Authors (Dan Bornstein et alia). +// SPDX-License-Identifier: Apache-2.0 + +import * as fs from 'node:fs'; + +// +// Argument parsing +// + +const { mainModule, modulesDirs } = JSON.parse(process.argv[2]); + + +// +// Helper functions +// + +/** + * Indicates whether or not the path corresponds to a regular file which is + * readable. + * + * @param {string} path The path to check. + * @returns {boolean} `true` iff the `path` is a readable file. + */ +function canRead(path) { + try { + fs.accessSync(path, fs.constants.R_OK); + const stats = fs.statSync(path); + return stats.isFile(); + } catch { + return false; + } +} + +/** + * "Pops" an arbitrary item from a `Set`. + * + * @param {Set} set The set to pop from. + * @returns {*} The popped item. + */ +function setPop(set) { + for (const item of set) { + set.delete(item); + return item; + } + + throw new Error('Empty `Set`.'); +} + +/** + * Sorts the entries of a plain object by key, returning a sorted version. + * + * @param {object} orig Object to sort. + * @returns {object} Sorted version. + */ +function sortObject(orig) { + const result = {}; + + for (const key of Object.keys(orig).sort()) { + result[key] = orig[key]; + } + + return result; +} + + +// +// Main script +// + +const errors = []; + +// Collect all of the modules referenced by this package, transitively including +// all referenced local modules. This uses a work queue arrangement where we +// start with the main subproject as the sole element of the to-be-processed +// queue. + +/** The names of all as-yet unprocessed local modules. */ +const unprocessed = new Set([`@this/${mainModule}`]); + +/** The names of all already-processed local modules. */ +const processed = new Set(); + +/** The graph of local module dependencies, as a list of edges. */ +let graph = []; + +/** Map from external dependency names to sets of all encountered versions. */ +const extDeps = new Map(); + +/** The names of all local modules encountered as dependencies. */ +const localDeps = new Set(); + +/** The path to each local module directory. */ +const localDirs = new Map(); + +while (unprocessed.size > 0) { + const oneDep = setPop(unprocessed); + + processed.add(oneDep); + localDeps.add(oneDep); + + // Trim `@this/` off of `oneDep`. + const oneDepName = oneDep.match(/(?<=[/])[^/]+$/)?.[0]; + + if (!oneDepName) { + errors.push(`Could not parse module name: ${oneDep}`); + continue; + } + + let moduleDir = null; + let packageObj = null; + for (const dir of modulesDirs) { + const fullDir = `${dir}/${oneDepName}`; + const pkgFile = `${fullDir}/package.json`; + + if (canRead(pkgFile)) { + moduleDir = fullDir; + packageObj = JSON.parse(fs.readFileSync(pkgFile)); + break; + } + } + + if (!moduleDir) { + errors.push(`Could not find module: ${oneDep}`); + continue; + } + + localDirs.set(oneDep, moduleDir); + + for (const [key, value] of Object.entries(packageObj.dependencies ?? {})) { + if (key.startsWith('@this/')) { + if (!processed.has(key)) { + unprocessed.add(key); + } + graph.push({ from: oneDep, to: key }); + } else { + let extSet = extDeps.get(key); + if (!extSet) { + extSet = new Set(); + extDeps.set(key, extSet); + } + extSet.add(value); + } + } +} + +// Build up the final result. + +const result = { + main: `@this/${mainModule}`, + localDeps: [...localDeps].sort(), + localDirs: sortObject(Object.fromEntries(localDirs.entries())), + extDeps: sortObject(Object.fromEntries(extDeps.entries())) +}; + +// `extDeps` has sets for values. Reduce them to single elements, and report an +// error for any item with multiple values. + +for (const [key, value] of Object.entries(result.extDeps)) { + if (value.size !== 1) { + errors.push(`Conflicting versions of external dependency \`${key}\`:`); + for (const v of value) { + errors.push(` ${v}`); + } + } else { + result.extDeps[key] = setPop(value); + } +} + +// Verify that the local module dependency graph has no cycles. If there's at +// least one cycle, list all the modules involved with cycles. +// +// What's going on: We start with the full directed graph, and iteratively +// remove all edges for nodes that only appear on the `from` side (because de +// facto they are not involved in a cycle). Once no more edges can be removed, +// any remaining ones are involved in cycles. + +const fromNodes = new Set(graph.map(({ from }) => from)); + +for (;;) { + const toNodes = new Set(graph.map(({ to }) => to)); + let anyRemoved = false; + + for (const f of fromNodes) { + if (!toNodes.has(f)) { + graph = graph.filter(({ from, to }) => (from !== f)); + fromNodes.delete(f); + anyRemoved = true; + } + } + + if (anyRemoved) { + continue; + } + + // Check for self-dependency. If found, report the error, remove the nodes, + // and keep checking. + for (const { from, to } of graph) { + if (from === to) { + errors.push(`Local module self-dependency: ${from}`); + graph = graph.filter(({ from: f }) => (from !== f)); + fromNodes.delete(from); + anyRemoved = true; + } + } + + if (!anyRemoved) { + break; + } +} + +if (graph.length !== 0) { + errors.push('Local module dependency cycle(s) detected.'); + errors.push('Modules involved:'); + for (const f of fromNodes) { + errors.push(` ${f}`); + } +} + +// Either report errors or return the final result. + +if (errors.length !== 0) { + for (const error of errors) { + process.stderr.write(`${error}\n`); + } + process.exit(1); +} + +console.log(JSON.stringify(result, null, 2));