Skip to content

Commit

Permalink
Added centralized config utilization in Audit for CI Next (#146)
Browse files Browse the repository at this point in the history
  • Loading branch information
eranturgeman authored Sep 4, 2024
1 parent 8c43a37 commit e2976fb
Show file tree
Hide file tree
Showing 13 changed files with 403 additions and 46 deletions.
2 changes: 1 addition & 1 deletion commands/audit/audit.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityPa
if err != nil {
return fmt.Errorf("failed to create jas scanner: %s", err.Error())
}
if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform()); err != nil {
if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform(), auditParams.configProfile); err != nil {
return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error())
}
return
Expand Down
151 changes: 151 additions & 0 deletions commands/audit/audit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package audit

import (
biutils "github.com/jfrog/build-info-go/utils"
"github.com/jfrog/jfrog-cli-core/v2/common/format"
coreTests "github.com/jfrog/jfrog-cli-core/v2/utils/tests"
"github.com/jfrog/jfrog-cli-security/utils"
"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
clientTests "github.com/jfrog/jfrog-client-go/utils/tests"
"github.com/jfrog/jfrog-client-go/xsc/services"
"github.com/stretchr/testify/assert"
"os"
"path/filepath"
"testing"
)

// Note: Currently, if a config profile is provided, the scan will use the profile's settings, IGNORING jfrog-apps-config if exists.
func TestAuditWithConfigProfile(t *testing.T) {
testcases := []struct {
name string
configProfile services.ConfigProfile
expectedSastIssues int
expectedSecretsIssues int
}{
{
name: "Enable only secrets scanner",
configProfile: services.ConfigProfile{
ProfileName: "only-secrets",
Modules: []services.Module{{
ModuleId: 1,
ModuleName: "only-secrets-module",
PathFromRoot: ".",
ScanConfig: services.ScanConfig{
SastScannerConfig: services.SastScannerConfig{
EnableSastScan: false,
},
SecretsScannerConfig: services.SecretsScannerConfig{
EnableSecretsScan: true,
},
},
}},
IsDefault: false,
},
expectedSastIssues: 0,
expectedSecretsIssues: 7,
},
{
name: "Enable only sast scanner",
configProfile: services.ConfigProfile{
ProfileName: "only-sast",
Modules: []services.Module{{
ModuleId: 1,
ModuleName: "only-sast-module",
PathFromRoot: ".",
ScanConfig: services.ScanConfig{
SastScannerConfig: services.SastScannerConfig{
EnableSastScan: true,
},
SecretsScannerConfig: services.SecretsScannerConfig{
EnableSecretsScan: false,
},
},
}},
IsDefault: false,
},
expectedSastIssues: 1,
expectedSecretsIssues: 0,
},
{
name: "Enable secrets and sast",
configProfile: services.ConfigProfile{
ProfileName: "secrets&sast",
Modules: []services.Module{{
ModuleId: 1,
ModuleName: "secrets&sast-module",
PathFromRoot: ".",
ScanConfig: services.ScanConfig{
SastScannerConfig: services.SastScannerConfig{
EnableSastScan: true,
},
SecretsScannerConfig: services.SecretsScannerConfig{
EnableSecretsScan: true,
},
},
}},
IsDefault: false,
},
expectedSastIssues: 1,
expectedSecretsIssues: 7,
},
}

for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
mockServer, serverDetails := utils.XrayServer(t, utils.EntitlementsMinVersion)
defer mockServer.Close()

auditBasicParams := (&utils.AuditBasicParams{}).
SetServerDetails(serverDetails).
SetOutputFormat(format.Table).
SetUseJas(true)

configProfile := testcase.configProfile
auditParams := NewAuditParams().
SetGraphBasicParams(auditBasicParams).
SetConfigProfile(&configProfile).
SetCommonGraphScanParams(&scangraph.CommonGraphScanParams{
RepoPath: "",
ProjectKey: "",
Watches: nil,
ScanType: "dependency",
IncludeVulnerabilities: true,
XscVersion: services.ConfigProfileMinXscVersion,
MultiScanId: "random-msi",
})
auditParams.SetIsRecursiveScan(true)

tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
defer createTempDirCallback()
testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))

baseWd, err := os.Getwd()
assert.NoError(t, err)
chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath)
defer chdirCallback()

auditResults, err := RunAudit(auditParams)
assert.NoError(t, err)

// Currently, the only supported scanners are Secrets and Sast, therefore if a config profile is utilized - all other scanners are disabled.
if testcase.expectedSastIssues > 0 {
assert.NotNil(t, auditResults.ExtendedScanResults.SastScanResults)
assert.Equal(t, testcase.expectedSastIssues, len(auditResults.ExtendedScanResults.SastScanResults[0].Results))
} else {
assert.Nil(t, auditResults.ExtendedScanResults.SastScanResults)
}

if testcase.expectedSecretsIssues > 0 {
assert.NotNil(t, auditResults.ExtendedScanResults.SecretsScanResults)
assert.Equal(t, testcase.expectedSecretsIssues, len(auditResults.ExtendedScanResults.SecretsScanResults[0].Results))
} else {
assert.Nil(t, auditResults.ExtendedScanResults.SecretsScanResults)
}

assert.Nil(t, auditResults.ScaResults)
assert.Nil(t, auditResults.ExtendedScanResults.ApplicabilityScanResults)
assert.Nil(t, auditResults.ExtendedScanResults.IacScanResults)
})
}
}
7 changes: 7 additions & 0 deletions commands/audit/auditparams.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/jfrog/jfrog-cli-security/utils/severityutils"
"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
"github.com/jfrog/jfrog-client-go/xray/services"
clientservices "github.com/jfrog/jfrog-client-go/xsc/services"
)

type AuditParams struct {
Expand All @@ -19,6 +20,7 @@ type AuditParams struct {
// Include third party dependencies source code in the applicability scan.
thirdPartyApplicabilityScan bool
threads int
configProfile *clientservices.ConfigProfile
}

func NewAuditParams() *AuditParams {
Expand Down Expand Up @@ -92,6 +94,11 @@ func (params *AuditParams) SetCommonGraphScanParams(commonParams *scangraph.Comm
return params
}

func (params *AuditParams) SetConfigProfile(configProfile *clientservices.ConfigProfile) *AuditParams {
params.configProfile = configProfile
return params
}

func (params *AuditParams) createXrayGraphScanParams() *services.XrayGraphScanParams {
return &services.XrayGraphScanParams{
RepoPath: params.commonGraphScanParams.RepoPath,
Expand Down
5 changes: 5 additions & 0 deletions commands/audit/scarunner.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ func buildDepTreeAndRunScaScan(auditParallelRunner *utils.SecurityParallelRunner
log.Debug("Skipping SCA scan as requested by input...")
return
}
if auditParams.configProfile != nil {
log.Debug("Skipping SCA scan as a configuration profile is being utilized and currently only Secrets and Sast scanners are supported when utilizing a configuration profile")
return
}

// Prepare
currentWorkingDir, err := os.Getwd()
if errorutils.CheckError(err) != nil {
Expand Down
2 changes: 1 addition & 1 deletion commands/scan/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo
log.Error(fmt.Sprintf("failed to create jas scanner: %s", err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
}
err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, &depsList, scanCmd.serverDetails, false, scanner, applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc, utils.GetAllSupportedScans())
err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, &depsList, scanCmd.serverDetails, false, scanner, applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc, utils.GetAllSupportedScans(), nil)
if err != nil {
log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
Expand Down
37 changes: 31 additions & 6 deletions jas/runner/jasrunner.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package runner

import (
"fmt"

"github.com/jfrog/gofrog/parallel"
jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
Expand All @@ -15,11 +14,13 @@ import (
"github.com/jfrog/jfrog-cli-security/utils/jasutils"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/log"
"github.com/jfrog/jfrog-client-go/xsc/services"
"golang.org/x/exp/slices"
)

func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, scanResults *utils.Results, directDependencies *[]string,
serverDetails *config.ServerDetails, thirdPartyApplicabilityScan bool, scanner *jas.JasScanner, scanType applicability.ApplicabilityScanType, secretsScanType secrets.SecretsScanType, errHandlerFunc func(error), scansToPreform []utils.SubScanType) (err error) {
serverDetails *config.ServerDetails, thirdPartyApplicabilityScan bool, scanner *jas.JasScanner, scanType applicability.ApplicabilityScanType,
secretsScanType secrets.SecretsScanType, errHandlerFunc func(error), scansToPreform []utils.SubScanType, configProfile *services.ConfigProfile) (err error) {
if serverDetails == nil || len(serverDetails.Url) == 0 {
log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.")
return
Expand All @@ -31,27 +32,51 @@ func AddJasScannersTasks(securityParallelRunner *utils.SecurityParallelRunner, s
}
// Set environments variables for analytics in analyzers manager.
// Don't execute other scanners when scanning third party dependencies.
// Currently, if config profile exists, the only possible scanners to run are: Secrets, Sast
if !thirdPartyApplicabilityScan {
for _, module := range scanner.JFrogAppsConfig.Modules {
if len(scansToPreform) > 0 && !slices.Contains(scansToPreform, utils.SecretsScan) {
log.Debug("Skipping secrets scan as requested by input...")
} else if configProfile != nil {
// This code section is related to CentralizedConfig integration in CI Next.
log.Debug(fmt.Sprintf("Using config profile '%s' to determine whether to run secrets scan...", configProfile.ProfileName))
if configProfile.Modules[0].ScanConfig.SecretsScannerConfig.EnableSecretsScan {
err = addModuleJasScanTask(jfrogappsconfig.Module{}, jasutils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc)
} else {
log.Debug(fmt.Sprintf("Skipping secrets scan as requested by '%s' config profile...", configProfile.ProfileName))
}
} else if err = addModuleJasScanTask(module, jasutils.Secrets, securityParallelRunner, runSecretsScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module, secretsScanType), errHandlerFunc); err != nil {
return
}
if runAllScanners {
if len(scansToPreform) > 0 && !slices.Contains(scansToPreform, utils.IacScan) {
log.Debug("Skipping Iac scan as requested by input...")
} else if err = addModuleJasScanTask(module, jasutils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil {
return
if configProfile == nil {
if len(scansToPreform) > 0 && !slices.Contains(scansToPreform, utils.IacScan) {
log.Debug("Skipping Iac scan as requested by input...")
} else if err = addModuleJasScanTask(module, jasutils.IaC, securityParallelRunner, runIacScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil {
return
}
}
if len(scansToPreform) > 0 && !slices.Contains(scansToPreform, utils.SastScan) {
log.Debug("Skipping Sast scan as requested by input...")
} else if configProfile != nil {
log.Debug(fmt.Sprintf("Using config profile '%s' to determine whether to run Sast scan...", configProfile.ProfileName))
if configProfile.Modules[0].ScanConfig.SastScannerConfig.EnableSastScan {
err = addModuleJasScanTask(jfrogappsconfig.Module{}, jasutils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc)
} else {
log.Debug(fmt.Sprintf("Skipping Sast scan as requested by '%s' config profile...", configProfile.ProfileName))
}
} else if err = addModuleJasScanTask(module, jasutils.Sast, securityParallelRunner, runSastScan(securityParallelRunner, scanner, scanResults.ExtendedScanResults, module), errHandlerFunc); err != nil {
return
}
}
}
}

if configProfile != nil {
log.Debug("Config profile is in use. Skipping Contextual Analysis scan as it is not currently supported with a config profile...")
return
}

if len(scansToPreform) > 0 && !slices.Contains(scansToPreform, utils.ContextualAnalysisScan) {
log.Debug("Skipping contextual analysis scan as requested by input...")
return err
Expand Down
2 changes: 1 addition & 1 deletion jas/runner/jasrunner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) {
scanner := &jas.JasScanner{}
jasScanner, err := jas.CreateJasScanner(scanner, nil, &jas.FakeServerDetails, jas.GetAnalyzerManagerXscEnvVars("", scanResults.GetScaScannedTechnologies()...))
assert.NoError(t, err)
err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, false, jasScanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan, utils.GetAllSupportedScans())
err = AddJasScannersTasks(securityParallelRunnerForTest, scanResults, &[]string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, false, jasScanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, securityParallelRunnerForTest.AddErrorToChan, utils.GetAllSupportedScans(), nil)
assert.NoError(t, err)
}

Expand Down
49 changes: 49 additions & 0 deletions tests/testdata/other/configProfile/configProfileExample.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"profile_name": "default-profile",
"frogbot_config": {
"email_author": "[email protected]",
"aggregate_fixes": true,
"avoid_previous_pr_comments_deletion": true,
"branch_name_template": "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}",
"pr_title_template": "[🐸 Frogbot] Upgrade {IMPACTED_PACKAGE} to {FIX_VERSION}",
"pr_comment_title": "Frogbot notes:",
"commit_message_template": "Upgrade {IMPACTED_PACKAGE} to {FIX_VERSION}",
"show_secrets_as_pr_comment": false
},
"modules": [
{
"module_name": "default-module",
"path_from_root": ".",
"releases_repo": "nuget-remote",
"analyzer_manager_version": "1.8.1",
"additional_paths_for_module": ["lib1", "utils/lib2"],
"exclude_paths": ["**/.git/**", "**/*test*/**", "**/*venv*/**", "**/*node_modules*/**", "**/target/**"],
"scan_config": {
"scan_timeout": 600,
"exclude_pattern": "*.md",
"enable_sca_scan": true,
"enable_contextual_analysis_scan": true,
"sast_scanner_config": {
"enable_sast_scan": true
},
"secrets_scanner_config": {
"enable_secrets_scan": true
},
"iac_scanner_config": {
"enable_iac_scan": true
},
"applications_scanner_config": {
"enable_applications_scan": true
},
"services_scanner_config": {
"enable_services_scan": true
}
},
"protected_branches": ["main", "master"],
"include_exclude_mode": 0,
"include_exclude_pattern": "*test*",
"report_analytics": true
}
],
"is_default": true
}
Loading

0 comments on commit e2976fb

Please sign in to comment.