Add support to calculate npm tree only by package-lock.json file (#194)

build/npm.go

@@ -74,7 +74,8 @@ func (nm *NpmModule) CalcDependencies() error {
 	if !nm.containingBuild.buildNameAndNumberProvided() {
 		return errors.New("a build name must be provided in order to collect the project's dependencies")
 	}
-	buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name, nm.npmArgs, true, nm.containingBuild.logger)
+	buildInfoDependencies, err := buildutils.CalculateNpmDependenciesList(nm.executablePath, nm.srcPath, nm.name,
+		buildutils.NpmTreeDepListParam{Args: nm.npmArgs}, true, nm.containingBuild.logger)
 	if err != nil {
 		return err
 	}

build/testdata/npm/project6/package-lock_test.json

@@ -0,0 +1,36 @@
+{
+  "name": "project6",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "project6",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "lightweight": "^0.1.0",
+        "minimist": "^0.1.0",
+        "underscore": "^1.13.6"
+      }
+    },
+    "node_modules/lightweight": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/lightweight/-/lightweight-0.1.0.tgz",
+      "integrity": "sha512-10pYSQA9EJqZZnXDR0urhg8Z0Y1XnRfi41ZFj3ZFTKJ5PjRq82HzT7LKlPyxewy3w2WA2POfi3jQQn7Y53oPcQ==",
+      "bin": {
+        "lwt": "bin/lwt.js"
+      }
+    },
+    "node_modules/minimist": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz",
+      "integrity": "sha512-wR5Ipl99t0mTGwLjQJnBjrP/O7zBbLZqvA3aw32DmLx+nXHfWctUjzDjnDx09pX1Po86WFQazF9xUzfMea3Cnw=="
+    },
+    "node_modules/underscore": {
+      "version": "1.13.6",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
+      "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
+    }
+  }
+}

build/testdata/npm/project6/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "npm_test2",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "lightweight": "^0.1.0",
+    "minimist": "^0.1.0",
+    "underscore": "^1.13.6",
+    "cors.js": "0.0.1-security"
+  }
+}

build/utils/npm.go

@@ -17,19 +17,19 @@ import (
 )
 
 // CalculateNpmDependenciesList gets an npm project's dependencies.
-func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmArgs []string, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) {
+func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) {
 	if log == nil {
 		log = &utils.NullLog{}
 	}
 	// Calculate npm dependency tree using 'npm ls...'.
-	dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmArgs, log)
+	dependenciesMap, err := CalculateDependenciesMap(executablePath, srcPath, moduleId, npmParams, log)
 	if err != nil {
 		return nil, err
 	}
 	var cacache *cacache
 	if calculateChecksums {
 		// Get local npm cache.
-		cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmArgs, log)
+		cacheLocation, err := GetNpmConfigCache(srcPath, executablePath, npmParams.Args, log)
 		if err != nil {
 			return nil, err
 		}
@@ -87,7 +87,7 @@ type dependencyInfo struct {
 
 // Run 'npm list ...' command and parse the returned result to create a dependencies map of.
 // The dependencies map looks like name:version -> entities.Dependency.
-func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs []string, log utils.Log) (map[string]*dependencyInfo, error) {
+func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log) (map[string]*dependencyInfo, error) {
 	dependenciesMap := make(map[string]*dependencyInfo)
 	// These arguments must be added at the end of the command, to override their other values (if existed in nm.npmArgs).
 	npmVersion, err := GetNpmVersion(executablePath, log)
@@ -100,10 +100,10 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmArgs
 	}
 	var data []byte
 	// If we don't have node_modules, the function will use the package-lock dependencies.
-	if nodeModulesExist {
-		data = runNpmLsWithNodeModules(executablePath, srcPath, npmArgs, log)
+	if nodeModulesExist && !npmListParams.IgnoreNodeModules {
+		data = runNpmLsWithNodeModules(executablePath, srcPath, npmListParams.Args, log)
 	} else {
-		data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmArgs, log, npmVersion)
+		data, err = runNpmLsWithoutNodeModules(executablePath, srcPath, npmListParams, log, npmVersion)
 		if err != nil {
 			return nil, err
 		}
@@ -130,19 +130,19 @@ func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, l
 	return
 }
 
-func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]byte, error) {
+func runNpmLsWithoutNodeModules(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, error) {
 	isPackageLockExist, isDirExistsErr := utils.IsFileExists(filepath.Join(srcPath, "package-lock.json"), false)
 	if isDirExistsErr != nil {
 		return nil, isDirExistsErr
 	}
-	if !isPackageLockExist {
-		err := installPackageLock(executablePath, srcPath, npmArgs, log, npmVersion)
+	if !isPackageLockExist || (npmListParams.OverwritePackageLock && checkIfLockFileShouldBeUpdated(srcPath, log)) {
+		err := installPackageLock(executablePath, srcPath, npmListParams.Args, log, npmVersion)
 		if err != nil {
 			return nil, err
 		}
 	}
-	npmArgs = append(npmArgs, "--json", "--all", "--long", "--package-lock-only")
-	data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log)
+	npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only")
+	data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmListParams.Args, "ls"), log)
 	if err != nil {
 		log.Warn(err.Error())
 	} else if len(errData) > 0 {
@@ -164,6 +164,25 @@ func installPackageLock(executablePath, srcPath string, npmArgs []string, log ut
 	return errors.New("it looks like you’re using version " + npmVersion.GetVersion() + " of the npm client. Versions below 6.0.0 require running `npm install` before running this command")
 }
 
+// Check if package.json has been modified.
+// This might indicate the addition of new packages to package.json that haven't been reflected in package-lock.json.
+func checkIfLockFileShouldBeUpdated(srcPath string, log utils.Log) bool {
+	packageJsonInfo, err := os.Stat(filepath.Join(srcPath, "package.json"))
+	if err != nil {
+		log.Warn("Failed to get file info for package.json, err: %v", err)
+		return false
+	}
+
+	packageJsonInfoModTime := packageJsonInfo.ModTime()
+	packageLockInfo, err := os.Stat(filepath.Join(srcPath, "package-lock.json"))
+	if err != nil {
+		log.Warn("Failed to get file info for package-lock.json, err: %v", err)
+		return false
+	}
+	packageLockInfoModTime := packageLockInfo.ModTime()
+	return packageJsonInfoModTime.After(packageLockInfoModTime)
+}
+
 func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, error) {
 	versionData, _, err := RunNpmCmd(executablePath, "", []string{"--version"}, log)
 	if err != nil {
@@ -172,6 +191,14 @@ func GetNpmVersion(executablePath string, log utils.Log) (*version.Version, erro
 	return version.NewVersion(string(versionData)), nil
 }
 
+type NpmTreeDepListParam struct {
+	Args []string
+	// Ignore the node_modules folder if exists, using the '--package-lock-only' flag
+	IgnoreNodeModules bool
+	// Rewrite package-lock.json, if exists.
+	OverwritePackageLock bool
+}
+
 // npm >=7 ls results for a single dependency
 type npmLsDependency struct {
 	Name      string

build/utils/npm_test.go

@@ -1,10 +1,13 @@
 package utils
 
 import (
+	"github.com/jfrog/build-info-go/entities"
+	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
+	"time"
 
 	testdatautils "github.com/jfrog/build-info-go/build/testdata"
 	"github.com/jfrog/build-info-go/utils"
@@ -192,12 +195,91 @@ func TestDependencyWithNoIntegrity(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "jfrogtest", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
 }
 
+// This test case verifies that CalculateNpmDependenciesList correctly handles the exclusion of 'node_modules'
+// and updates 'package-lock.json' as required, based on the 'IgnoreNodeModules' and 'OverwritePackageLock' parameters.
+func TestDependencyPackageLockOnly(t *testing.T) {
+	npmVersion, _, err := GetNpmVersionAndExecPath(logger)
+	require.NoError(t, err)
+	if !npmVersion.AtLeast("7.0.0") {
+		t.Skip("Running on npm v7 and above only, skipping...")
+	}
+	path, cleanup := testdatautils.CreateTestProject(t, filepath.Join("..", "testdata/npm/project6"))
+	defer cleanup()
+	assert.NoError(t, utils.MoveFile(filepath.Join(path, "package-lock_test.json"), filepath.Join(path, "package-lock.json")))
+	// sleep so the package.json modified time will be bigger than the package-lock.json, this make sure it will recalculate lock file.
+	require.NoError(t, os.Chtimes(filepath.Join(path, "package.json"), time.Now(), time.Now().Add(time.Millisecond*20)))
+
+	// Calculate dependencies.
+	dependencies, err := CalculateDependenciesMap("npm", path, "jfrogtest",
+		NpmTreeDepListParam{Args: []string{}, IgnoreNodeModules: true, OverwritePackageLock: true}, logger)
+	assert.NoError(t, err)
+	var expectedRes = getExpectedRespForTestDependencyPackageLockOnly()
+	assert.Equal(t, expectedRes, dependencies)
+}
+
+func getExpectedRespForTestDependencyPackageLockOnly() map[string]*dependencyInfo {
+	return map[string]*dependencyInfo{
+		"underscore:1.13.6": {
+			Dependency: entities.Dependency{
+				Id:          "underscore:1.13.6",
+				Scopes:      []string{"prod"},
+				RequestedBy: [][]string{{"jfrogtest"}},
+				Checksum:    entities.Checksum{},
+			},
+			npmLsDependency: &npmLsDependency{
+				Name:      "underscore",
+				Version:   "1.13.6",
+				Integrity: "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==",
+			},
+		},
+		"cors.js:0.0.1-security": {
+			Dependency: entities.Dependency{
+				Id:          "cors.js:0.0.1-security",
+				Scopes:      []string{"prod"},
+				RequestedBy: [][]string{{"jfrogtest"}},
+				Checksum:    entities.Checksum{},
+			},
+			npmLsDependency: &npmLsDependency{
+				Name:      "cors.js",
+				Version:   "0.0.1-security",
+				Integrity: "sha512-Cu4D8imt82jd/AuMBwTpjrXiULhaMdig2MD2NBhRKbbcuCTWeyN2070SCEDaJuI/4kA1J9Nnvj6/cBe/zfnrrw==",
+			},
+		},
+		"lightweight:0.1.0": {
+			Dependency: entities.Dependency{
+				Id:          "lightweight:0.1.0",
+				Scopes:      []string{"prod"},
+				RequestedBy: [][]string{{"jfrogtest"}},
+				Checksum:    entities.Checksum{},
+			},
+			npmLsDependency: &npmLsDependency{
+				Name:      "lightweight",
+				Version:   "0.1.0",
+				Integrity: "sha512-10pYSQA9EJqZZnXDR0urhg8Z0Y1XnRfi41ZFj3ZFTKJ5PjRq82HzT7LKlPyxewy3w2WA2POfi3jQQn7Y53oPcQ==",
+			},
+		},
+		"minimist:0.1.0": {
+			Dependency: entities.Dependency{
+				Id:          "minimist:0.1.0",
+				Scopes:      []string{"prod"},
+				RequestedBy: [][]string{{"jfrogtest"}},
+				Checksum:    entities.Checksum{},
+			},
+			npmLsDependency: &npmLsDependency{
+				Name:      "minimist",
+				Version:   "0.1.0",
+				Integrity: "sha512-wR5Ipl99t0mTGwLjQJnBjrP/O7zBbLZqvA3aw32DmLx+nXHfWctUjzDjnDx09pX1Po86WFQazF9xUzfMea3Cnw==",
+			},
+		},
+	}
+}
+
 // A project built differently for each operating system.
 func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) {
 	npmVersion, _, err := GetNpmVersionAndExecPath(logger)
@@ -214,15 +296,15 @@ func TestDependenciesTreeDifferentBetweenOKs(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "bundle-dependencies", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
 
 	// Remove node_modules directory, then calculate dependencies by package-lock.
 	assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules")))
 
-	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	// Asserting there is at least one dependency.
@@ -253,7 +335,7 @@ func TestNpmProdFlag(t *testing.T) {
 			assert.NoError(t, err)
 
 			// Calculate dependencies with scope.
-			dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+			dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 			assert.NoError(t, err)
 			assert.Len(t, dependencies, entry.totalDeps)
 		}()
@@ -302,15 +384,15 @@ func validateDependencies(t *testing.T, projectPath string, npmArgs []string) {
 	assert.NoError(t, err)
 
 	// Calculate dependencies.
-	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err := CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	assert.Greaterf(t, len(dependencies), 0, "Error: dependencies are not found!")
 
 	// Remove node_modules directory, then calculate dependencies by package-lock.
 	assert.NoError(t, utils.RemoveTempDir(filepath.Join(projectPath, "node_modules")))
 
-	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", npmArgs, true, logger)
+	dependencies, err = CalculateNpmDependenciesList("npm", projectPath, "build-info-go-tests", NpmTreeDepListParam{Args: npmArgs}, true, logger)
 	assert.NoError(t, err)
 
 	// Asserting there is at least one dependency.