Add support to calculate npm tree only by package-lock.json file

build/npm.go

@@ -74,7 +74,8 @@ func (nm *NpmModule) CalcDependencies() error {
 	if !nm.containingBuild.buildNameAndNumberProvided() {
 		return errors.New("a build name must be provided in order to collect the project's dependencies")
 	}
-	buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name, nm.npmArgs, true, nm.containingBuild.logger)
+	buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name,
+		buildutils.NpmTreeDepListParam{Args: nm.npmArgs}, true, nm.containingBuild.logger)
 	if err != nil {
 		return err
 	}

build/utils/npm.go

@@ -17,19 +17,19 @@ import (
 )
 
 // CalculateNpmDependenciesList gets an npm project's dependencies.
-func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmArgs []string, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) {
+func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) {
 	if log == nil {
 		log = &utils.NullLog{}
 	}
 	// Calculate npm dependency tree using 'npm ls...'.
-	dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmArgs, log)
+	dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmParams, log)
 	if err != nil {
 		return nil, err
 	}
 	var cacache *cacache
 	if calculateChecksums {
 		// Get local npm cache.
-		cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmArgs, log)
+		cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmParams.Args, log)
 		if err != nil {
 			return nil, err
 		}
@@ -87,7 +87,7 @@ type dependencyInfo struct {
 
 // Run 'npm list ...' command and parse the returned result to create a dependencies map of.
 // The dependencies map looks like name:version -> entities.Dependency.
-func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs []string, log utils.Log) (map[string]*dependencyInfo, error) {
+func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log) (map[string]*dependencyInfo, error) {
 	dependenciesMap := make(map[string]*dependencyInfo)
 	// These arguments must be added at the end of the command, to override their other values (if existed in nm.npmArgs).
 	npmVersion, err := GetNpmVersion(executablePath, log)
@@ -100,10 +100,10 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs
 	}
 	var data []byte
 	// If we don't have node_modules, the function will use the package-lock dependencies.
-	if nodeModulesExist {
-		data = runNpmLsWithNodeModules(executablePath, srcPath, npmArgs, log)
+	if nodeModulesExist && !npmListParams.IgnoreNodeModules {
+		data = runNpmLsWithNodeModules(executablePath, srcPath, npmListParams.Args, log)
 	} else {
-		data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmArgs, log, npmVersion)
+		data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmListParams, log, npmVersion)
 		if err != nil {
 			return nil, err
 		}
@@ -130,19 +130,19 @@ func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, l
 	return
 }
 
-func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]byte, error) {
+func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, error) {
 	isPackageLockExist, isDirExistsErr := utils.IsFileExists(filepath.Join(srcPath, "package-lock.json"), false)
 	if isDirExistsErr != nil {
 		return nil, isDirExistsErr
 	}
-	if !isPackageLockExist {
-		err := installPackageLock(executablePath, srcPath, npmArgs, log, npmVersion)
+	if !isPackageLockExist || (npmListParams.OverWritePackageLock && checkIfLockFileShouldBeUpdated(srcPath, log)) {
+		err := installPackageLock(executablePath, srcPath, npmListParams.Args, log, npmVersion)
 		if err != nil {
 			return nil, err
 		}
 	}
-	npmArgs = append(npmArgs, "--json", "--all", "--long", "--package-lock-only")
-	data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log)
+	npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only")
+	data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmListParams.Args, "ls"), log)
 	if err != nil {
 		log.Warn(err.Error())
 	} else if len(errData) > 0 {
@@ -164,6 +164,25 @@ func installPackageLock(executablePath, srcPath string, npmArgs []string, log ut
 	return errors.New("it looks like you’re using version " + npmVersion.GetVersion() + " of the npm client. Versions below 6.0.0 require running `npm install` before running this command")
 }
 
+// check package.json modified,
+// this can hint us new packages added to package.json which were not updated in package-lock.json..
+func checkIfLockFileShouldBeUpdated(srcPath string, log utils.Log) bool {
+	packageJsonInfo, err := os.Stat(filepath.Join(srcPath, "package.json"))
+	if err != nil {
+		log.Warn("Failed to get file info for package.json, err: %v", err)
+		return false
+	}
+
+	packageJsonInfoModTime := packageJsonInfo.ModTime()
+	packageLockInfo, err := os.Stat(filepath.Join(srcPath, "package-lock.json"))
+	if err != nil {
+		log.Warn("Failed to get file info for package-lock.json, err: %v", err)
+		return false
+	}
+	packageLockInfoModTime := packageLockInfo.ModTime()
+	return packageJsonInfoModTime.After(packageLockInfoModTime)
+}
+
 func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, error) {
 	versionData, _, err := RunNpmCmd(executablePath, "", []string{"--version"}, log)
 	if err != nil {
@@ -172,6 +191,14 @@ func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, erro
 	return version.NewVersion(string(versionData)), nil
 }
 
+type NpmTreeDepListParam struct {
+	Args []string
+	// ignore node modules folder if exists.
+	IgnoreNodeModules bool
+	// rewrite package lock json if exists.
+	OverWritePackageLock bool
+}
+
 // npm >=7 ls results for a single dependency
 type npmLsDependency struct {
 	Name      string

build/utils/npm_test.go

@@ -192,7 +192,7 @@ func TestDependencyWithNoIntegrity(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
@@ -214,15 +214,15 @@ func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
 
 	// Remove node_modules directory, then calculate dependencies by package-lock.
 	assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules")))
 
-	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	// Asserting there is at least one dependency.
@@ -253,7 +253,7 @@ func TestNpmProdFlag(t *testing.T) {
 			assert.NoError(t, err)
 
 			// Calculate dependencies with scope.
-			dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+			dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 			assert.NoError(t, err)
 			assert.Len(t, dependencies, entry.totalDeps)
 		}()
@@ -302,15 +302,15 @@ func validateDependencies(t *testing.T, projectPath string, npmArgs []string) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
 
 	// Remove node_modules directory, then calculate dependencies by package-lock.
 	assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules")))
 
-	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	// Asserting there is at least one dependency.