Skip to content

Commit

Permalink
Add support for new contextual scan statuses (#36)
Browse files Browse the repository at this point in the history
  • Loading branch information
orz25 authored Apr 15, 2024
1 parent ba26e60 commit c0ab022
Show file tree
Hide file tree
Showing 10 changed files with 478 additions and 99 deletions.
14 changes: 7 additions & 7 deletions audit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,14 +173,14 @@ func TestXrayAuditNugetJson(t *testing.T) {
projectName: "multi",
format: string(format.Json),
restoreTech: "dotnet",
minVulnerabilities: 5,
minVulnerabilities: 4,
minLicences: 3,
},
{
projectName: "multi",
format: string(format.Json),
restoreTech: "",
minVulnerabilities: 5,
minVulnerabilities: 4,
minLicences: 3,
},
}
Expand Down Expand Up @@ -329,7 +329,7 @@ func TestXrayAuditMultiProjects(t *testing.T) {
defer securityTestUtils.CleanTestsHomeEnv()
output := securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.SimpleJson), workingDirsFlag)
securityTestUtils.VerifySimpleJsonScanResults(t, output, 35, 0)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 0, 0, 0, 25, 1)
}

func TestXrayAuditPipJson(t *testing.T) {
Expand Down Expand Up @@ -433,18 +433,18 @@ func addDummyPackageDescriptor(t *testing.T, hasPackageJson bool) {

func TestXrayAuditJasSimpleJson(t *testing.T) {
output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas-test"))
securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 2)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 1, 9, 7, 3, 0, 2, 2)
}

func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) {
output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("jas", "jas-config"))
securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 1, 2)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 1, 3, 0, 2, 2)
}

func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) {
output := testXrayAuditJas(t, string(format.SimpleJson), filepath.Join("package-managers", "npm", "npm"))
securityTestUtils.VerifySimpleJsonScanResults(t, output, 1, 0)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 0, 0)
securityTestUtils.VerifySimpleJsonJasResults(t, output, 0, 0, 0, 0, 0, 0, 1)
}

func testXrayAuditJas(t *testing.T, format string, project string) string {
Expand Down Expand Up @@ -509,7 +509,7 @@ func TestXrayRecursiveScan(t *testing.T) {
output := securityTests.PlatformCli.RunCliCmdWithOutput(t, "audit", "--format=json")

// We anticipate the identification of five vulnerabilities: four originating from the .NET project and one from the NPM project.
securityTestUtils.VerifyJsonScanResults(t, output, 0, 5, 0)
securityTestUtils.VerifyJsonScanResults(t, output, 0, 4, 0)

var results []services.ScanResponse
err = json.Unmarshal([]byte(output), &results)
Expand Down
84 changes: 44 additions & 40 deletions commands/audit/jas/applicability/applicabilitymanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,54 +283,58 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) {
assert.True(t, len(fileContent) > 0)
}

func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) {
// Arrange
scanner, cleanUp := jas.InitJasTest(t)
defer cleanUp()

applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false)
applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "empty-results.sarif")

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix)

if assert.NoError(t, err) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
assert.Empty(t, applicabilityManager.applicabilityScanResults[0].Results)
}
}

func TestParseResults_ApplicableCveExist(t *testing.T) {
// Arrange
scanner, cleanUp := jas.InitJasTest(t)
defer cleanUp()
applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false)
applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "applicable-cve-results.sarif")

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix)
func TestParseResults_NewApplicabilityStatuses(t *testing.T) {
testCases := []struct {
name string
fileName string
expectedResults int
expectedApplicabilityStatuses []string
}{
{
name: "empty results - all cves should get unknown",
fileName: "empty-results.sarif",
expectedResults: 0,
},
{
name: "applicable cve exist",
fileName: "applicable-cve-results.sarif",
expectedResults: 2,
},
{
name: "all cves not applicable",
fileName: "no-applicable-cves-results.sarif",
expectedResults: 5,
},

if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
assert.NotEmpty(t, applicabilityManager.applicabilityScanResults[0].Results)
{
name: "new applicability statuses",
fileName: "new_ca_status.sarif",
expectedResults: 5,
expectedApplicabilityStatuses: []string{"applicable", "undetermined", "not_covered", "not_applicable"},
},
}
}

func TestParseResults_AllCvesNotApplicable(t *testing.T) {
// Arrange
scanner, cleanUp := jas.InitJasTest(t)
defer cleanUp()
applicabilityManager := newApplicabilityScanManager(jas.FakeBasicXrayResults, mockDirectDependencies, scanner, false)
applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", "no-applicable-cves-results.sarif")

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix)

if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
assert.NotEmpty(t, applicabilityManager.applicabilityScanResults[0].Results)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
applicabilityManager.scanner.ResultsFileName = filepath.Join(jas.GetTestDataPath(), "applicability-scan", tc.fileName)
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix)
if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
assert.Len(t, applicabilityManager.applicabilityScanResults[0].Results, tc.expectedResults)
if tc.name == "new applicability statuses" {
assert.Len(t, applicabilityManager.applicabilityScanResults[0].Tool.Driver.Rules, len(tc.expectedApplicabilityStatuses))
for i, value := range tc.expectedApplicabilityStatuses {
assert.Equal(t, value, applicabilityManager.applicabilityScanResults[0].Tool.Driver.Rules[i].Properties["applicability"])
}
}
}
})
}
}
2 changes: 1 addition & 1 deletion scangraph/scangraph_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func TestFilterResultIfNeeded(t *testing.T) {
},
},
params: ScanGraphParams{
severityLevel: 11,
severityLevel: 15,
},
expected: services.ScanResponse{
Violations: []services.Violation{
Expand Down
187 changes: 187 additions & 0 deletions tests/testdata/other/applicability-scan/new_ca_status.sarif
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
{
"runs": [
{
"tool": {
"driver": {
"name": "JFrog Applicability Scanner",
"rules": [
{
"id": "applic_CVE-2020-14343",
"name": "CVE-2020-14343",
"properties": {
"conclusion": "negative",
"applicability": "applicable"
},
"fullDescription": {
"text": "The scanner checks whether any of the following vulnerable functions are called:\n\n- `yaml.full_load()`\n- `yaml.load()` only unsafe calls (without specifying `SafeLoader` as the `Loader`class).",
"markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n- `yaml.full_load()`\n- `yaml.load()` only unsafe calls (without specifying `SafeLoader` as the `Loader`class)."
},
"shortDescription": {
"text": "Scanner for CVE-2020-14343"
}
},
{
"id": "applic_CVE-2020-1751",
"name": "CVE-2020-1751",
"properties": {
"conclusion": "UNSEEN",
"applicability": "undetermined"
}
},
{
"id": "applic_CVE-2020-1750",
"name": "CVE-2020-1750",
"properties": {
"applicability": "not_covered"
},
"fullDescription": {
"text": "The scanner checks whether any of the following vulnerable functions are called:\n\n- `yaml.full_load()`\n- `yaml.load()` only unsafe calls (without specifying `SafeLoader` as the `Loader`class).",
"markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n- `yaml.full_load()`\n- `yaml.load()` only unsafe calls (without specifying `SafeLoader` as the `Loader`class)."
},
"shortDescription": {
"text": "Scanner for CVE-2020-1747"
}
},
{
"id": "applic_CVE-2020-7788",
"name": "CVE-2020-7788",
"properties": {
"conclusion": "positive",
"applicability": "not_applicable"
},
"fullDescription": {
"text": "The scanner checks whether the vulnerable function `ini.parse` is called.",
"markdown": "The scanner checks whether the vulnerable function `ini.parse` is called."
},
"shortDescription": {
"text": "Scanner for CVE-2020-7788"
}
}
],
"version": "1.0",
"informationUri": "https://jfrog.com/help/r/jfrog-security-documentation/jfrog-advanced-security"
}
},
"invocations": [
{
"arguments": [
"/var/root/.jfrog/dependencies/analyzerManager/ca_scanner/applicability_scanner",
"scan",
"/tmp/jfrog.cli.temp.-1709245721-1151253638/config.yaml"
],
"executionSuccessful": true,
"workingDirectory": {
"uri": "file:///private/var/root/.jfrog/dependencies/analyzerManager"
}
}
],
"results": [
{
"message": {
"text": "The vulnerable function yaml.full_load/load is called"
},
"locations": [
{
"physicalLocation": {
"region": {
"snippet": {
"text": "yaml.full_load(f)"
},
"endColumn": 28,
"endLine": 4,
"startColumn": 11,
"startLine": 4
},
"artifactLocation": {
"uri": "file:///tmp/tmpv7yob8g_/unpacked/filesystem/blobs/sha256/dcdf8bff5b7e55b7c546dfd6085997ac9e0cc5e3c2cfe3639999dac4bc3e678e/applicable/main.py"
}
}
}
],
"ruleId": "applic_CVE-2020-14343"
},
{
"message": {
"text": "The vulnerable function yaml.full_load/load is called"
},
"locations": [
{
"physicalLocation": {
"region": {
"snippet": {
"text": "yaml.load(f, Loader=yaml.FullLoader)"
},
"endColumn": 47,
"endLine": 4,
"startColumn": 11,
"startLine": 4
},
"artifactLocation": {
"uri": "file:///tmp/tmpv7yob8g_/unpacked/filesystem/blobs/sha256/dcdf8bff5b7e55b7c546dfd6085997ac9e0cc5e3c2cfe3639999dac4bc3e678e/applicable/main2.py"
}
}
}
],
"ruleId": "applic_CVE-2020-14343"
},
{
"message": {
"text": "The vulnerable function yaml.full_load/load is called"
},
"locations": [
{
"physicalLocation": {
"region": {
"snippet": {
"text": "yaml.full_load(f)"
},
"endColumn": 28,
"endLine": 4,
"startColumn": 11,
"startLine": 4
},
"artifactLocation": {
"uri": "file:///tmp/tmpv7yob8g_/unpacked/filesystem/blobs/sha256/dcdf8bff5b7e55b7c546dfd6085997ac9e0cc5e3c2cfe3639999dac4bc3e678e/applicable/main.py"
}
}
}
],
"ruleId": "applic_CVE-2020-1747"
},
{
"message": {
"text": "The vulnerable function yaml.full_load/load is called"
},
"locations": [
{
"physicalLocation": {
"region": {
"snippet": {
"text": "yaml.load(f, Loader=yaml.FullLoader)"
},
"endColumn": 47,
"endLine": 4,
"startColumn": 11,
"startLine": 4
},
"artifactLocation": {
"uri": "file:///tmp/tmpv7yob8g_/unpacked/filesystem/blobs/sha256/dcdf8bff5b7e55b7c546dfd6085997ac9e0cc5e3c2cfe3639999dac4bc3e678e/applicable/main2.py"
}
}
}
],
"ruleId": "applic_CVE-2020-1747"
},
{
"message": {
"text": "The scanner checks whether the vulnerable function `ini.parse` is called."
},
"kind": "pass",
"ruleId": "applic_CVE-2020-7788"
}
]
}
],
"version": "2.1.0",
"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json"
}
18 changes: 13 additions & 5 deletions tests/utils/test_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,22 +64,30 @@ func VerifySimpleJsonScanResults(t *testing.T, content string, minVulnerabilitie
}
}

func VerifySimpleJsonJasResults(t *testing.T, content string, minSastViolations, minIacViolations, minSecrets, minApplicable int) {
func VerifySimpleJsonJasResults(t *testing.T, content string, minSastViolations, minIacViolations, minSecrets,
minApplicable, minUndetermined, minNotCovered, minNotApplicable int) {
var results formats.SimpleJsonResults
err := json.Unmarshal([]byte(content), &results)
if assert.NoError(t, err) {
assert.GreaterOrEqual(t, len(results.Sast), minSastViolations, "Found less sast then expected")
assert.GreaterOrEqual(t, len(results.Secrets), minSecrets, "Found less secrets then expected")
assert.GreaterOrEqual(t, len(results.Iacs), minIacViolations, "Found less IaC then expected")
var applicableResults, notApplicableResults int
var applicableResults, undeterminedResults, notCoveredResults, notApplicableResults int
for _, vuln := range results.Vulnerabilities {
if vuln.Applicable == string(utils.NotApplicable) {
switch vuln.Applicable {
case string(utils.NotApplicable):
notApplicableResults++
} else if vuln.Applicable == string(utils.Applicable) {
case string(utils.Applicable):
applicableResults++
case string(utils.NotCovered):
notCoveredResults++
case string(utils.ApplicabilityUndetermined):
undeterminedResults++
}
}
assert.GreaterOrEqual(t, applicableResults, minApplicable, "Found less applicableResults then expected")
assert.GreaterOrEqual(t, notApplicableResults, 1, "Found less notApplicableResults then expected")
assert.GreaterOrEqual(t, undeterminedResults, minUndetermined, "Found less undeterminedResults then expected")
assert.GreaterOrEqual(t, notCoveredResults, minNotCovered, "Found less notCoveredResults then expected")
assert.GreaterOrEqual(t, notApplicableResults, minNotApplicable, "Found less notApplicableResults then expected")
}
}
1 change: 1 addition & 0 deletions utils/analyzermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
Applicable ApplicabilityStatus = "Applicable"
NotApplicable ApplicabilityStatus = "Not Applicable"
ApplicabilityUndetermined ApplicabilityStatus = "Undetermined"
NotCovered ApplicabilityStatus = "Not Covered"
NotScanned ApplicabilityStatus = ""
)

Expand Down
Loading

0 comments on commit c0ab022

Please sign in to comment.