From 76f4130dd41dd98df5d0e45c859ca9899e37e67a Mon Sep 17 00:00:00 2001 From: Ke Wu Date: Thu, 14 Sep 2023 19:17:04 +0800 Subject: [PATCH] feat: add npm package-lock.json support (#462) Co-authored-by: tianding.wk --- .gitignore | 1 + bin/install.js | 24 ++++++ lib/download/npm.js | 18 ++++- lib/lockfile_resolver.js | 94 ++++++++++++++++++++++++ test/fixtures/lockfile/package-lock.json | 85 +++++++++++++++++++++ test/fixtures/lockfile/package.json | 9 +++ test/install-with-lockfile.test.js | 48 ++++++++++++ 7 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 lib/lockfile_resolver.js create mode 100644 test/fixtures/lockfile/package-lock.json create mode 100644 test/fixtures/lockfile/package.json create mode 100644 test/install-with-lockfile.test.js diff --git a/.gitignore b/.gitignore index 942222ae..edf255f1 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ test/fixtures/npm-workspaces/package.json test/fixtures/uninstall/package.json package-lock.json +!test/fixtures/lockfile/package-lock.json diff --git a/bin/install.js b/bin/install.js index 0d2c07ad..dae6cd69 100755 --- a/bin/install.js +++ b/bin/install.js @@ -20,6 +20,7 @@ const { ALIAS_TYPES, } = require('../lib/npa_types'); const Context = require('../lib/context'); +const { lockfileConverter } = require('../lib/lockfile_resolver'); const originalArgv = process.argv.slice(2); @@ -39,6 +40,18 @@ Object.assign(argv, parseArgs(originalArgv, { // npminstall foo --workspace=aa // npminstall foo -w aa 'workspace', + /** + * set package-lock.json path + * + * 1. only support package lock v2 and v3. + * 2. npminstall doesn't inspect /package-lock.json by default. + * 3. because arborist doesn't support client/build/isomorphic dependencies, + * these kinds of dependencies will all be ignored. + * 4. this option doesn't do extra check for the equivalence of package-lock.json and package.json + * simply behaves like `npm ci` but doesn't remove the node_modules in advance. + * 5. you're not supposed to install extra dependencies along with a lockfile. + */ + 'lockfile-path', ], boolean: [ 'version', @@ -118,6 +131,7 @@ Usage: npminstall npminstall / npminstall --proxy=http://localhost:8080 + npminstall --lockfile-path= Can specify one or more: npminstall ./foo.tgz bar@stable /some/folder If no argument is supplied, installs dependencies from ./package.json. @@ -344,6 +358,16 @@ debug('argv: %j, env: %j', argv, env); }; } + const lockfilePath = argv['lockfile-path']; + if (lockfilePath) { + try { + const lockfileData = await fs.readFile(lockfilePath, 'utf8'); + config.dependenciesTree = lockfileConverter(JSON.parse(lockfileData)); + } catch (error) { + console.warn(chalk.yellow('npminstall WARN load lockfile from %s error :%s'), lockfilePath, error.message); + } + } + const dependenciesTree = argv['dependencies-tree']; if (dependenciesTree) { try { diff --git a/lib/download/npm.js b/lib/download/npm.js index f8d1cb79..382ce696 100644 --- a/lib/download/npm.js +++ b/lib/download/npm.js @@ -715,7 +715,9 @@ const defaultExtensions = toMap([ function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options) { return new Promise((resolve, reject) => { const shasum = pkg.dist.shasum; - const hash = crypto.createHash('sha1'); + const integrity = pkg.dist.integrity; + const algorithmType = pkg.dist.checkSSRI ? 'sha512' : 'sha1'; + const hash = crypto.createHash(algorithmType); let tarballSize = 0; const opts = { cwd: ungzipDir, @@ -785,9 +787,17 @@ function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options) }); readstream.on('end', () => { // this will be fire before extracter `env` event fire. - const realShasum = hash.digest('hex'); - if (realShasum !== shasum) { - const err = new Error(`real sha1:${realShasum} not equal to remote:${shasum}, download url ${readstream.tarballUrl || ''}, download size ${tarballSize}`); + let hashResult = ''; + let hashString = ''; + if (pkg.dist.checkSSRI) { + hashResult = algorithmType + '-' + hash.digest('base64'); + hashString = integrity; + } else { + hashResult = hash.digest('hex'); + hashString = shasum; + } + if (hashResult !== hashString) { + const err = new Error(`real ${algorithmType}:${hashResult} not equal to remote:${hashString}, download url ${readstream.tarballUrl || ''}, download size ${tarballSize}`); err.name = 'ShasumNotMatchError'; handleCallback(err); } diff --git a/lib/lockfile_resolver.js b/lib/lockfile_resolver.js new file mode 100644 index 00000000..b0c865ad --- /dev/null +++ b/lib/lockfile_resolver.js @@ -0,0 +1,94 @@ +'use strict'; + +const path = require('node:path'); +const assert = require('node:assert'); +const dependencies = require('./dependencies'); + +const NODE_MODULES_DIR = 'node_modules/'; + +/** + * The lockfileConverter converts a npm package-lockfile.json to npminstall .dependencies-tree.json. + * Only lockfileVersion >= 2 is supported. + * The `.dependencies-tree.json` does not recognize a npm-workspaces package, so we don't need to handle it neither for now. + * @param {Object} lockfile package-lock.json data + * @param {Object} options installation options + * @param {Nested} nested Nested + */ +exports.lockfileConverter = function lockfileConverter(lockfile, options, nested) { + assert(lockfile.lockfileVersion >= 2, 'Only lockfileVersion >=2 is supported.'); + + const tree = {}; + + const packages = lockfile.packages; + for (const pkgPath in packages) { + /** + * the lockfile contains all the deps so there's no need to be deps type sensitive. + */ + const deps = dependencies(packages[pkgPath], options, nested); + const allMap = deps.allMap; + for (const key in allMap) { + const mani = exports.nodeModulesPath(pkgPath, key, packages); + const dist = { + checkSSRI: true, + integrity: mani.integrity, + tarball: mani.resolved, + }; + // we need to remove the integrity and resolved field from the mani + // but the mani should be left intact + const maniClone = Object.assign({}, mani); + delete maniClone.integrity; + delete maniClone.resolved; + tree[`${key}@${allMap[key]}`] = { + name: key, + ...maniClone, + dist, + _id: `${key}@${mani.version}`, + }; + } + } + + return tree; +}; + +/** + * find the matched version of current semver based on nodejs modules resolution algorithm + * we need check the current directory and ancestors directories to find the matched version of lodash.has + * e.g. + * "": { + * "dependencies": { + * "lodash.has": "4", + * "a": "latest" + * }, + * 'node_modules/lodash.has': { + * "version": '4.4.0' + * }, + * 'node_modules/a': { + * 'dependencies': { + * "lodash.has": "3" + * } + * }, + * 'node_modules/a/node_modules/lodash.has': { + * "version": "3.0.0" + * }, + * } + * + * the root lodash.has@4 should matches node_modules/lodash.has + * + * @param {string} currentPath the current pakcage.json path + * @param {string} name the dependency name of current package.json + * @param {Object} packages the lockfile packages + */ +exports.nodeModulesPath = function nodeModulesPath(currentPath, name, packages) { + const dirs = currentPath.split(NODE_MODULES_DIR); + dirs.push(name); + + do { + const dir = path.normalize('.' + dirs.join('/' + NODE_MODULES_DIR)); + const pkg = packages[dir]; + if (pkg) { + return pkg; + } + + dirs.splice(dirs.length - 2, 1); + } while (dirs.length > 1); +}; diff --git a/test/fixtures/lockfile/package-lock.json b/test/fixtures/lockfile/package-lock.json new file mode 100644 index 00000000..a79e1e77 --- /dev/null +++ b/test/fixtures/lockfile/package-lock.json @@ -0,0 +1,85 @@ +{ + "name": "lockfile", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "lodash._baseget": "3.7.1", + "lodash._baseslice": "^3.0.1", + "lodash.get": "3", + "lodash.has": "4.0.0", + "lodash.has3": "npm:lodash.has@3" + } + }, + "node_modules/lodash._baseget": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/lodash._baseget/-/lodash._baseget-3.7.1.tgz", + "integrity": "sha512-VFICYluEPRjeVlLDdg6tGNyYplqEC5aE3ZzN2mScC7aoD96mYYwzsalX8VLQFd7W0M9Vsp+JMp+TlDmIvVhXkA==" + }, + "node_modules/lodash._baseslice": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash._baseslice/-/lodash._baseslice-3.0.3.tgz", + "integrity": "sha512-DkeB6ZDkxIAkBvfuaq4ZHE8vN/FW0tZxMERQxci1zP0oFH6xC1ogpebPNgANXSwwcpjZPDAGWd4EFyATAf7Nsw==" + }, + "node_modules/lodash._topath": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/lodash._topath/-/lodash._topath-3.8.1.tgz", + "integrity": "sha512-QsF5c8A+Biv0oxuSCd05JqhXHPCjvFT0nMXVevfMgU1pp5iEHVSin2cKXi3lQe5+px285p7kAHVtOnbNE79syw==", + "dependencies": { + "lodash.isarray": "^3.0.0" + } + }, + "node_modules/lodash.get": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-3.7.0.tgz", + "integrity": "sha512-7iD0aRHu/B8gcCDNx53lJi33R4TzpbOB3Mfk4XpIN7WFUt+W5rI+6CtHhpJ52B6zhhRvogtuNSDFZc3xgcbClQ==", + "dependencies": { + "lodash._baseget": "^3.0.0", + "lodash._topath": "^3.0.0" + } + }, + "node_modules/lodash.has": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.0.0.tgz", + "integrity": "sha512-Xw3htH7ERWlsAkYbelC1I4ItQafDzLPP7h42GQjdxp4G7EyF7/gkEp1z3zGpw+6ceiKpDyq4lOX2dZqTk8qIfQ==", + "dependencies": { + "lodash._baseslice": "^4.0.0", + "lodash.get": "^4.0.0" + } + }, + "node_modules/lodash.has/node_modules/lodash._baseslice": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash._baseslice/-/lodash._baseslice-4.0.0.tgz", + "integrity": "sha512-gFMKGnEY/5cjMx2+x+yfKvrcm3w1PWWqUgWyo4r9xfTqVIaObWoTGoaIEp4ZWwEs/DQ3hoBzzW8W0Rus5+IcZw==" + }, + "node_modules/lodash.has/node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" + }, + "node_modules/lodash.has3": { + "name": "lodash.has", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-3.2.1.tgz", + "integrity": "sha512-KL1Z/fbG2omhWTc90h1NzS/JDsW+/0rCfoBUQBIBZCRnaJxFwagpm/dtQppzstlXXVvRy9jqCWBxoyeAkFxsaw==", + "dependencies": { + "lodash._baseget": "^3.0.0", + "lodash._baseslice": "^3.0.0", + "lodash._topath": "^3.0.0", + "lodash.isarguments": "^3.0.0", + "lodash.isarray": "^3.0.0" + } + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, + "node_modules/lodash.isarray": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz", + "integrity": "sha512-JwObCrNJuT0Nnbuecmqr5DgtuBppuCvGD9lxjFpAzwnVtdGoDQ1zig+5W8k5/6Gcn0gZ3936HDAlGd28i7sOGQ==" + } + } +} \ No newline at end of file diff --git a/test/fixtures/lockfile/package.json b/test/fixtures/lockfile/package.json new file mode 100644 index 00000000..adb92c6a --- /dev/null +++ b/test/fixtures/lockfile/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "lodash.has": "4.0.0", + "lodash.get": "3", + "lodash._baseget": "3.7.1", + "lodash._baseslice": "^3.0.1", + "lodash.has3": "npm:lodash.has@3" + } +} \ No newline at end of file diff --git a/test/install-with-lockfile.test.js b/test/install-with-lockfile.test.js new file mode 100644 index 00000000..fc626316 --- /dev/null +++ b/test/install-with-lockfile.test.js @@ -0,0 +1,48 @@ +const coffee = require('coffee'); +const path = require('node:path'); +const assert = require('node:assert'); +const fs = require('node:fs/promises'); +const { lockfileConverter } = require('../lib/lockfile_resolver'); +const Nested = require('../lib/nested'); +const helper = require('./helper'); + +describe('test/install-with-lockfile.test.js', () => { + const cwd = helper.fixtures('lockfile'); + const lockfile = require(path.join(cwd, 'package-lock.json')); + const nested = new Nested([]); + const cleanup = helper.cleanup(cwd); + + beforeEach(cleanup); + afterEach(cleanup); + + // the Windows path sucks, shamefully skip these tests + if (process.platform !== 'win32') { + it('should install successfully', async () => { + await coffee.fork( + helper.npminstall, + [ + '--lockfile-path', + path.join(cwd, 'package-lock.json'), + ], { cwd }) + .debug() + .expect('code', 0) + .end(); + assert.strictEqual( + await fs.readlink(path.join(cwd, 'node_modules', 'lodash.has3'), 'utf8'), + '.store/lodash.has@3.2.1/node_modules/lodash.has' + ); + assert.strictEqual( + await fs.readlink(path.join(cwd, 'node_modules', 'lodash.has'), 'utf8'), + '.store/lodash.has@4.0.0/node_modules/lodash.has' + ); + }); + + it('should convert package-lock.json to .dependencies-tree.json successfully', () => { + const dependenciesTree = lockfileConverter(lockfile, { + ignoreOptionalDependencies: true, + }, nested); + + assert.strictEqual(Object.keys(dependenciesTree).length, 12); + }); + } +});