Merge branch 'dev' into validate-xray-url

cli/docs/flags.go

@@ -116,6 +116,7 @@ const (
 	ThirdPartyContextualAnalysis = "third-party-contextual-analysis"
 	RequirementsFile             = "requirements-file"
 	WorkingDirs                  = "working-dirs"
+	OutputDir                    = "output-dir"
 
 	// Unique curation flags
 	CurationOutput = "curation-format"
@@ -152,7 +153,7 @@ var commandFlags = map[string][]string{
 		url, user, password, accessToken, ServerId, InsecureTls, Project, Watches, RepoPath, Licenses, OutputFormat, ExcludeTestDeps,
 		useWrapperAudit, DepType, RequirementsFile, Fail, ExtendedTable, WorkingDirs, ExclusionsAudit, Mvn, Gradle, Npm,
 		Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads,
-		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln,
+		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln, OutputDir,
 	},
 	CurationAudit: {
 		CurationOutput, WorkingDirs, Threads, RequirementsFile,
@@ -228,6 +229,7 @@ var flagsMap = map[string]components.Flag{
 		components.WithBoolDefaultValue(true),
 	),
 	WorkingDirs: components.NewStringFlag(WorkingDirs, "A comma-separated list of relative working directories, to determine audit targets locations."),
+	OutputDir:   components.NewStringFlag(OutputDir, "Target directory to save partial results to.", components.SetHiddenStrFlag()),
 	ExclusionsAudit: components.NewStringFlag(
 		Exclusions,
 		"List of exclusions separated by semicolons, utilized to skip sub-projects from undergoing an audit. These exclusions may incorporate the * and ? wildcards.",
@@ -259,10 +261,10 @@ var flagsMap = map[string]components.Flag{
 
 	// Git flags
 	InputFile:       components.NewStringFlag(InputFile, "Path to an input file in YAML format contains multiple git providers. With this option, all other scm flags will be ignored and only git servers mentioned in the file will be examined.."),
-	ScmType:         components.NewStringFlag(ScmType, fmt.Sprintf("SCM type. Possible values are: %s.", git.NewScmType().GetValidScmTypeString())),
-	ScmApiUrl:       components.NewStringFlag(ScmApiUrl, "SCM API URL. For example: 'https://api.github.com'."),
-	Token:           components.NewStringFlag(Token, fmt.Sprintf("SCM API token. In the absence of a flag, tokens should be passed in the %s enviroment variable, or in the corresponding environment variables '%s'.", git.GenericGitTokenEnvVar, git.NewScmType().GetOptionalScmTypeTokenEnvVars())),
-	Owner:           components.NewStringFlag(Owner, "The format of the owner key depends on the Git provider: On GitHub and GitLab, the owner is typically an individual or an organization, On Bitbucket, the owner can also be a project. In the case of a private instance on Bitbucket, the individual or organization name should be prefixed with '~'."),
+	ScmType:         components.NewStringFlag(ScmType, fmt.Sprintf("SCM type. Possible values are: %s.", git.NewScmType().GetValidScmTypeString()), components.SetMandatory()),
+	ScmApiUrl:       components.NewStringFlag(ScmApiUrl, "SCM API URL. For example: 'https://api.github.com'.", components.SetMandatory()),
+	Token:           components.NewStringFlag(Token, fmt.Sprintf("SCM API token. In the absence of a flag, tokens should be passed in the %s enviroment variable, or in the corresponding environment variables '%s'.", git.GenericGitTokenEnvVar, git.NewScmType().GetOptionalScmTypeTokenEnvVars()), components.SetMandatory()),
+	Owner:           components.NewStringFlag(Owner, "The format of the owner key depends on the Git provider: On GitHub and GitLab, the owner is typically an individual or an organization, On Bitbucket, the owner can also be a project. In the case of a private instance on Bitbucket, the individual or organization name should be prefixed with '~'.", components.SetMandatory()),
 	RepoName:        components.NewStringFlag(RepoName, "List of semicolon-separated(;) repositories names to analyze, If not provided all repositories related to the provided owner will be analyzed."),
 	Months:          components.NewStringFlag(Months, "Number of months to analyze.", components.WithIntDefaultValue(git.DefaultContContributorsMonths)),
 	DetailedSummary: components.NewBoolFlag(DetailedSummary, "Set to true to get a contributors detailed summary."),

cli/scancommands.go

@@ -19,6 +19,7 @@ import (
 	"github.com/jfrog/jfrog-cli-security/commands/enrich"
 	"github.com/jfrog/jfrog-cli-security/utils/xray"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/urfave/cli"
 	"os"
@@ -455,6 +456,11 @@ func CreateAuditCmd(c *components.Context) (*audit.AuditCommand, error) {
 	if err != nil {
 		return nil, err
 	}
+	scansOutputDir, err := getAndValidateOutputDirExistsIfProvided(c)
+	if err != nil {
+		return nil, err
+	}
+
 	auditCmd.SetAnalyticsMetricsService(xsc.NewAnalyticsMetricsService(serverDetails))
 
 	auditCmd.SetTargetRepoPath(addTrailingSlashToRepoPathIfNeeded(c)).
@@ -465,7 +471,8 @@ func CreateAuditCmd(c *components.Context) (*audit.AuditCommand, error) {
 		SetPrintExtendedTable(c.GetBoolFlagValue(flags.ExtendedTable)).
 		SetMinSeverityFilter(minSeverity).
 		SetFixableOnly(c.GetBoolFlagValue(flags.FixableOnly)).
-		SetThirdPartyApplicabilityScan(c.GetBoolFlagValue(flags.ThirdPartyContextualAnalysis))
+		SetThirdPartyApplicabilityScan(c.GetBoolFlagValue(flags.ThirdPartyContextualAnalysis)).
+		SetScansResultsOutputDir(scansOutputDir)
 
 	if c.GetStringFlagValue(flags.Watches) != "" {
 		auditCmd.SetWatches(splitByCommaAndTrim(c.GetStringFlagValue(flags.Watches)))
@@ -629,6 +636,21 @@ func getCurationCommand(c *components.Context) (*curation.CurationAuditCommand,
 	return curationAuditCommand, nil
 }
 
+func getAndValidateOutputDirExistsIfProvided(c *components.Context) (string, error) {
+	scansOutputDir := c.GetStringFlagValue(flags.OutputDir)
+	if scansOutputDir == "" {
+		return "", nil
+	}
+	exists, err := fileutils.IsDirExists(scansOutputDir, false)
+	if err != nil {
+		return "", err
+	}
+	if !exists {
+		return "", fmt.Errorf("output directory path for saving scans results was provided, but the directory doesn't exist: '%s'", scansOutputDir)
+	}
+	return scansOutputDir, nil
+}
+
 func DockerScanMockCommand() components.Command {
 	// Mock how the CLI handles docker commands:
 	// https://github.com/jfrog/jfrog-cli/blob/v2/buildtools/cli.go#L691

commands/audit/audit.go

@@ -3,19 +3,17 @@ package audit
 import (
 	"errors"
 	"fmt"
-
 	jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-security/jas"
 	"github.com/jfrog/jfrog-cli-security/jas/applicability"
 	"github.com/jfrog/jfrog-cli-security/jas/runner"
 	"github.com/jfrog/jfrog-cli-security/jas/secrets"
+	"github.com/jfrog/jfrog-cli-security/utils"
 	"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
 	"github.com/jfrog/jfrog-cli-security/utils/xsc"
 
-	"github.com/jfrog/jfrog-cli-security/jas"
-	"github.com/jfrog/jfrog-cli-security/utils"
-
 	xrayutils "github.com/jfrog/jfrog-cli-security/utils/xray"
 	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/xray"
@@ -115,7 +113,8 @@ func (auditCmd *AuditCommand) Run() (err error) {
 		SetGraphBasicParams(auditCmd.AuditBasicParams).
 		SetCommonGraphScanParams(auditCmd.CreateCommonGraphScanParams()).
 		SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan).
-		SetThreads(auditCmd.Threads)
+		SetThreads(auditCmd.Threads).
+		SetScansResultsOutputDir(auditCmd.scanResultsOutputDir)
 	auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions())
 
 	auditResults, err := RunAudit(auditParams)
@@ -256,7 +255,7 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityPa
 	if err != nil {
 		return fmt.Errorf("failed to create jas scanner: %s", err.Error())
 	}
-	if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform(), auditParams.configProfile); err != nil {
+	if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform(), auditParams.configProfile, auditParams.scanResultsOutputDir); err != nil {
 		return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error())
 	}
 	return

commands/audit/audit_test.go

@@ -6,11 +6,12 @@ import (
 	coreTests "github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/jfrog/jfrog-cli-security/utils"
 	"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
-	clientTests "github.com/jfrog/jfrog-client-go/utils/tests"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
+	scanservices "github.com/jfrog/jfrog-client-go/xray/services"
 	"github.com/jfrog/jfrog-client-go/xsc/services"
 	"github.com/stretchr/testify/assert"
-	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -95,36 +96,30 @@ func TestAuditWithConfigProfile(t *testing.T) {
 			mockServer, serverDetails := utils.XrayServer(t, utils.EntitlementsMinVersion)
 			defer mockServer.Close()
 
+			tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+			defer createTempDirCallback()
+			testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
+			assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
+
 			auditBasicParams := (&utils.AuditBasicParams{}).
 				SetServerDetails(serverDetails).
 				SetOutputFormat(format.Table).
 				SetUseJas(true)
 
 			configProfile := testcase.configProfile
 			auditParams := NewAuditParams().
+				SetWorkingDirs([]string{tempDirPath}).
 				SetGraphBasicParams(auditBasicParams).
 				SetConfigProfile(&configProfile).
 				SetCommonGraphScanParams(&scangraph.CommonGraphScanParams{
 					RepoPath:               "",
-					ProjectKey:             "",
-					Watches:                nil,
-					ScanType:               "dependency",
+					ScanType:               scanservices.Dependency,
 					IncludeVulnerabilities: true,
 					XscVersion:             services.ConfigProfileMinXscVersion,
 					MultiScanId:            "random-msi",
 				})
 			auditParams.SetIsRecursiveScan(true)
 
-			tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
-			defer createTempDirCallback()
-			testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
-			assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
-
-			baseWd, err := os.Getwd()
-			assert.NoError(t, err)
-			chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath)
-			defer chdirCallback()
-
 			auditResults, err := RunAudit(auditParams)
 			assert.NoError(t, err)
 
@@ -149,3 +144,53 @@ func TestAuditWithConfigProfile(t *testing.T) {
 		})
 	}
 }
+
+// This test tests audit flow when providing --output-dir flag
+func TestAuditWithScansOutputDir(t *testing.T) {
+	mockServer, serverDetails := utils.XrayServer(t, utils.EntitlementsMinVersion)
+	defer mockServer.Close()
+
+	outputDirPath, removeOutputDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+	defer removeOutputDirCallback()
+
+	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+	defer createTempDirCallback()
+	testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
+	assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
+
+	auditBasicParams := (&utils.AuditBasicParams{}).
+		SetServerDetails(serverDetails).
+		SetOutputFormat(format.Table).
+		SetUseJas(true)
+
+	auditParams := NewAuditParams().
+		SetWorkingDirs([]string{tempDirPath}).
+		SetGraphBasicParams(auditBasicParams).
+		SetCommonGraphScanParams(&scangraph.CommonGraphScanParams{
+			ScanType:               scanservices.Dependency,
+			IncludeVulnerabilities: true,
+			MultiScanId:            utils.TestScaScanId,
+		}).
+		SetScansResultsOutputDir(outputDirPath)
+	auditParams.SetIsRecursiveScan(true)
+
+	_, err := RunAudit(auditParams)
+	assert.NoError(t, err)
+
+	filesList, err := fileutils.ListFiles(outputDirPath, false)
+	assert.NoError(t, err)
+	assert.Len(t, filesList, 5)
+
+	var fileNamesWithoutSuffix []string
+	for _, fileName := range filesList {
+		// Removing <hash>.json suffix to so we can check by suffix all expected files exist
+		splitName := strings.Split(fileName, "_")
+		fileNamesWithoutSuffix = append(fileNamesWithoutSuffix, splitName[0])
+	}
+
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "sca"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "iac"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "sast"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "secrets"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "applicability"))
+}

commands/audit/auditparams.go

@@ -21,6 +21,7 @@ type AuditParams struct {
 	thirdPartyApplicabilityScan bool
 	threads                     int
 	configProfile               *clientservices.ConfigProfile
+	scanResultsOutputDir        string
 }
 
 func NewAuditParams() *AuditParams {
@@ -99,6 +100,11 @@ func (params *AuditParams) SetConfigProfile(configProfile *clientservices.Config
 	return params
 }
 
+func (params *AuditParams) SetScansResultsOutputDir(outputDir string) *AuditParams {
+	params.scanResultsOutputDir = outputDir
+	return params
+}
+
 func (params *AuditParams) createXrayGraphScanParams() *services.XrayGraphScanParams {
 	return &services.XrayGraphScanParams{
 		RepoPath:               params.commonGraphScanParams.RepoPath,