diff --git a/build/npm.go b/build/npm.go index 66d52e04..10d658f3 100644 --- a/build/npm.go +++ b/build/npm.go @@ -74,7 +74,8 @@ func (nm *NpmModule) CalcDependencies() error { if !nm.containingBuild.buildNameAndNumberProvided() { return errors.New("a build name must be provided in order to collect the project's dependencies") } - buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name, nm.npmArgs, true, nm.containingBuild.logger) + buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name, + buildutils.NpmTreeDepListParam{Args: nm.npmArgs}, true, nm.containingBuild.logger) if err != nil { return err } diff --git a/build/testdata/npm/project6/node_modules/dummy-file b/build/testdata/npm/project6/node_modules/dummy-file new file mode 100644 index 00000000..e69de29b diff --git a/build/testdata/npm/project6/package-lock_test.json b/build/testdata/npm/project6/package-lock_test.json new file mode 100644 index 00000000..28c7bf77 --- /dev/null +++ b/build/testdata/npm/project6/package-lock_test.json @@ -0,0 +1,36 @@ +{ + "name": "project6", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "project6", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "lightweight": "^0.1.0", + "minimist": "^0.1.0", + "underscore": "^1.13.6" + } + }, + "node_modules/lightweight": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lightweight/-/lightweight-0.1.0.tgz", + "integrity": "sha512-10pYSQA9EJqZZnXDR0urhg8Z0Y1XnRfi41ZFj3ZFTKJ5PjRq82HzT7LKlPyxewy3w2WA2POfi3jQQn7Y53oPcQ==", + "bin": { + "lwt": "bin/lwt.js" + } + }, + "node_modules/minimist": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz", + "integrity": "sha512-wR5Ipl99t0mTGwLjQJnBjrP/O7zBbLZqvA3aw32DmLx+nXHfWctUjzDjnDx09pX1Po86WFQazF9xUzfMea3Cnw==" + }, + "node_modules/underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" + } + } +} diff --git a/build/testdata/npm/project6/package.json b/build/testdata/npm/project6/package.json new file mode 100644 index 00000000..9a4a6cbb --- /dev/null +++ b/build/testdata/npm/project6/package.json @@ -0,0 +1,17 @@ +{ + "name": "npm_test2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "lightweight": "^0.1.0", + "minimist": "^0.1.0", + "underscore": "^1.13.6", + "cors.js": "0.0.1-security" + } +} diff --git a/build/utils/npm.go b/build/utils/npm.go index f841488b..d1b81233 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -17,19 +17,19 @@ import ( ) // CalculateNpmDependenciesList gets an npm project's dependencies. -func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmArgs []string, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) { +func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) { if log == nil { log = &utils.NullLog{} } // Calculate npm dependency tree using 'npm ls...'. - dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmArgs, log) + dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmParams, log) if err != nil { return nil, err } var cacache *cacache if calculateChecksums { // Get local npm cache. - cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmArgs, log) + cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmParams.Args, log) if err != nil { return nil, err } @@ -87,7 +87,7 @@ type dependencyInfo struct { // Run 'npm list ...' command and parse the returned result to create a dependencies map of. // The dependencies map looks like name:version -> entities.Dependency. -func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs []string, log utils.Log) (map[string]*dependencyInfo, error) { +func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log) (map[string]*dependencyInfo, error) { dependenciesMap := make(map[string]*dependencyInfo) // These arguments must be added at the end of the command, to override their other values (if existed in nm.npmArgs). npmVersion, err := GetNpmVersion(executablePath, log) @@ -100,10 +100,10 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs } var data []byte // If we don't have node_modules, the function will use the package-lock dependencies. - if nodeModulesExist { - data = runNpmLsWithNodeModules(executablePath, srcPath, npmArgs, log) + if nodeModulesExist && !npmListParams.IgnoreNodeModules { + data = runNpmLsWithNodeModules(executablePath, srcPath, npmListParams.Args, log) } else { - data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmArgs, log, npmVersion) + data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmListParams, log, npmVersion) if err != nil { return nil, err } @@ -130,19 +130,19 @@ func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, l return } -func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]byte, error) { +func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, error) { isPackageLockExist, isDirExistsErr := utils.IsFileExists(filepath.Join(srcPath, "package-lock.json"), false) if isDirExistsErr != nil { return nil, isDirExistsErr } - if !isPackageLockExist { - err := installPackageLock(executablePath, srcPath, npmArgs, log, npmVersion) + if !isPackageLockExist || (npmListParams.OverwritePackageLock && checkIfLockFileShouldBeUpdated(srcPath, log)) { + err := installPackageLock(executablePath, srcPath, npmListParams.Args, log, npmVersion) if err != nil { return nil, err } } - npmArgs = append(npmArgs, "--json", "--all", "--long", "--package-lock-only") - data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log) + npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only") + data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmListParams.Args, "ls"), log) if err != nil { log.Warn(err.Error()) } else if len(errData) > 0 { @@ -164,6 +164,25 @@ func installPackageLock(executablePath, srcPath string, npmArgs []string, log ut return errors.New("it looks like you’re using version " + npmVersion.GetVersion() + " of the npm client. Versions below 6.0.0 require running `npm install` before running this command") } +// Check if package.json has been modified. +// This might indicate the addition of new packages to package.json that haven't been reflected in package-lock.json. +func checkIfLockFileShouldBeUpdated(srcPath string, log utils.Log) bool { + packageJsonInfo, err := os.Stat(filepath.Join(srcPath, "package.json")) + if err != nil { + log.Warn("Failed to get file info for package.json, err: %v", err) + return false + } + + packageJsonInfoModTime := packageJsonInfo.ModTime() + packageLockInfo, err := os.Stat(filepath.Join(srcPath, "package-lock.json")) + if err != nil { + log.Warn("Failed to get file info for package-lock.json, err: %v", err) + return false + } + packageLockInfoModTime := packageLockInfo.ModTime() + return packageJsonInfoModTime.After(packageLockInfoModTime) +} + func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, error) { versionData, _, err := RunNpmCmd(executablePath, "", []string{"--version"}, log) if err != nil { @@ -172,6 +191,14 @@ func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, erro return version.NewVersion(string(versionData)), nil } +type NpmTreeDepListParam struct { + Args []string + // Ignore the node_modules folder if exists, using the '--package-lock-only' flag + IgnoreNodeModules bool + // Rewrite package-lock.json, if exists. + OverwritePackageLock bool +} + // npm >=7 ls results for a single dependency type npmLsDependency struct { Name string diff --git a/build/utils/npm_test.go b/build/utils/npm_test.go index fc995b30..23509790 100644 --- a/build/utils/npm_test.go +++ b/build/utils/npm_test.go @@ -1,10 +1,13 @@ package utils import ( + "github.com/jfrog/build-info-go/entities" + "github.com/stretchr/testify/require" "os" "path/filepath" "reflect" "testing" + "time" testdatautils "github.com/jfrog/build-info-go/build/testdata" "github.com/jfrog/build-info-go/utils" @@ -192,12 +195,91 @@ func TestDependencyWithNoIntegrity(t *testing.T) { assert.NoError(t, err) // Calculate dependencies. - dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", npmArgs, true, logger) + dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!") } +// This test case verifies that CalculateNpmDependenciesList correctly handles the exclusion of 'node_modules' +// and updates 'package-lock.json' as required, based on the 'IgnoreNodeModules' and 'OverwritePackageLock' parameters. +func TestDependencyPackageLockOnly(t *testing.T) { + npmVersion, _, err := GetNpmVersionAndExecPath(logger) + require.NoError(t, err) + if !npmVersion.AtLeast("7.0.0") { + t.Skip("Running on npm v7 and above only, skipping...") + } + path, cleanup := testdatautils.CreateTestProject(t, filepath.Join("..", "testdata/npm/project6")) + defer cleanup() + assert.NoError(t, utils.MoveFile(filepath.Join(path, "package-lock_test.json"), filepath.Join(path, "package-lock.json"))) + // sleep so the package.json modified time will be bigger than the package-lock.json, this make sure it will recalculate lock file. + require.NoError(t, os.Chtimes(filepath.Join(path, "package.json"), time.Now(), time.Now().Add(time.Millisecond*20))) + + // Calculate dependencies. + dependencies, err := CalculateDependenciesMap("npm", path, "jfrogtest", + NpmTreeDepListParam{Args: []string{}, IgnoreNodeModules: true, OverwritePackageLock: true}, logger) + assert.NoError(t, err) + var expectedRes = getExpectedRespForTestDependencyPackageLockOnly() + assert.Equal(t, expectedRes, dependencies) +} + +func getExpectedRespForTestDependencyPackageLockOnly() map[string]*dependencyInfo { + return map[string]*dependencyInfo{ + "underscore:1.13.6": { + Dependency: entities.Dependency{ + Id: "underscore:1.13.6", + Scopes: []string{"prod"}, + RequestedBy: [][]string{{"jfrogtest"}}, + Checksum: entities.Checksum{}, + }, + npmLsDependency: &npmLsDependency{ + Name: "underscore", + Version: "1.13.6", + Integrity: "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", + }, + }, + "cors.js:0.0.1-security": { + Dependency: entities.Dependency{ + Id: "cors.js:0.0.1-security", + Scopes: []string{"prod"}, + RequestedBy: [][]string{{"jfrogtest"}}, + Checksum: entities.Checksum{}, + }, + npmLsDependency: &npmLsDependency{ + Name: "cors.js", + Version: "0.0.1-security", + Integrity: "sha512-Cu4D8imt82jd/AuMBwTpjrXiULhaMdig2MD2NBhRKbbcuCTWeyN2070SCEDaJuI/4kA1J9Nnvj6/cBe/zfnrrw==", + }, + }, + "lightweight:0.1.0": { + Dependency: entities.Dependency{ + Id: "lightweight:0.1.0", + Scopes: []string{"prod"}, + RequestedBy: [][]string{{"jfrogtest"}}, + Checksum: entities.Checksum{}, + }, + npmLsDependency: &npmLsDependency{ + Name: "lightweight", + Version: "0.1.0", + Integrity: "sha512-10pYSQA9EJqZZnXDR0urhg8Z0Y1XnRfi41ZFj3ZFTKJ5PjRq82HzT7LKlPyxewy3w2WA2POfi3jQQn7Y53oPcQ==", + }, + }, + "minimist:0.1.0": { + Dependency: entities.Dependency{ + Id: "minimist:0.1.0", + Scopes: []string{"prod"}, + RequestedBy: [][]string{{"jfrogtest"}}, + Checksum: entities.Checksum{}, + }, + npmLsDependency: &npmLsDependency{ + Name: "minimist", + Version: "0.1.0", + Integrity: "sha512-wR5Ipl99t0mTGwLjQJnBjrP/O7zBbLZqvA3aw32DmLx+nXHfWctUjzDjnDx09pX1Po86WFQazF9xUzfMea3Cnw==", + }, + }, + } +} + // A project built differently for each operating system. func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) { npmVersion, _, err := GetNpmVersionAndExecPath(logger) @@ -214,7 +296,7 @@ func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) { assert.NoError(t, err) // Calculate dependencies. - dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", npmArgs, true, logger) + dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!") @@ -222,7 +304,7 @@ func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) { // Remove node_modules directory, then calculate dependencies by package-lock. assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules"))) - dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger) + dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) // Asserting there is at least one dependency. @@ -253,7 +335,7 @@ func TestNpmProdFlag(t *testing.T) { assert.NoError(t, err) // Calculate dependencies with scope. - dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger) + dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) assert.Len(t, dependencies, entry.totalDeps) }() @@ -302,7 +384,7 @@ func validateDependencies(t *testing.T, projectPath string, npmArgs []string) { assert.NoError(t, err) // Calculate dependencies. - dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger) + dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!") @@ -310,7 +392,7 @@ func validateDependencies(t *testing.T, projectPath string, npmArgs []string) { // Remove node_modules directory, then calculate dependencies by package-lock. assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules"))) - dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger) + dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger) assert.NoError(t, err) // Asserting there is at least one dependency.