diff --git a/Makefile b/Makefile index 6326b509a..7cb8fcaf8 100644 --- a/Makefile +++ b/Makefile @@ -330,13 +330,13 @@ unit-tests: deps test-prep e2e-tests: deps test-prep @echo "+ $@" set -o pipefail ; \ - go test -tags e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log + go test -tags e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log .PHONY: slim-e2e-tests slim-e2e-tests: deps test-prep @echo "+ $@" set -o pipefail ; \ - go test -tags slim_e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log + go test -tags slim_e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log .PHONY: db-integration-tests db-integration-tests: deps test-prep diff --git a/e2etests/sanity_test.go b/e2etests/sanity_test.go index e05823091..d9c1c3106 100644 --- a/e2etests/sanity_test.go +++ b/e2etests/sanity_test.go @@ -20,18 +20,40 @@ func checkMatch(t *testing.T, source string, expectedVuln, matchingVuln v1.Vulne if expectedVuln.Metadata == nil { assert.Nil(t, matchingVuln.Metadata, "Expected no metadata for %s but got some", expectedVuln.Name) } else { - for _, keys := range [][]string{ + keySets := [][]string{ {source, "CVSSv2", "ExploitabilityScore"}, {source, "CVSSv2", "Score"}, {source, "CVSSv2", "ImpactScore"}, {source, "CVSSv2", "Vectors"}, - {source, "CVSSv3", "ExploitabilityScore"}, - {source, "CVSSv3", "Score"}, - {source, "CVSSv3", "ImpactScore"}, - {source, "CVSSv3", "Vectors"}, - } { - assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v", keys) - assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v", keys) + } + + testHasCVSSv3 := deepGet(expectedVuln.Metadata, []string{source, "CVSSv3"}...) != nil + + vulnCVSSv3 := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3"}...) + vulnCVSSv3Vectors := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3", "Vectors"}...) + // Assume that CVSSv3 scores exist if Vectors is not empty. + vulnHasCVSSv3 := vulnCVSSv3Vectors != nil && vulnCVSSv3Vectors.(string) != "" + + // If the test case does not have CVSSv3 scores but the matching vuln does, fail the test. + // This was added when NVD stopped returning CVSSv3 data for some vulns which we had + // test cases for, this condition ensures we are alerted when/if the data returns. + assert.False(t, !testHasCVSSv3 && vulnHasCVSSv3, "Test case for %q is missing CVSSv3 scores, please add the scores to the test case. Scores from vuln data: %+v", expectedVuln.Name, vulnCVSSv3) + + // Compare CVSSv3 data only when it exists in the test case. + if testHasCVSSv3 { + keySets = append(keySets, [][]string{ + {source, "CVSSv3", "ExploitabilityScore"}, + {source, "CVSSv3", "Score"}, + {source, "CVSSv3", "ImpactScore"}, + {source, "CVSSv3", "Vectors"}, + }...) + } else { + t.Logf("WARN: No CVSSv3 data provided for %q, skipping CVSSv3 field validation.", expectedVuln.Name) + } + + for _, keys := range keySets { + assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v in vuln %q", keys, expectedVuln.Name) + assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v in vuln %q", keys, expectedVuln.Name) } } expectedVuln.Metadata = nil diff --git a/e2etests/testcase_test.go b/e2etests/testcase_test.go index 323752bf9..5a9cb18e0 100644 --- a/e2etests/testcase_test.go +++ b/e2etests/testcase_test.go @@ -1204,12 +1204,13 @@ var testCases = []testCase{ "Score": 4.3, "Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.8, - "ImpactScore": 3.6, - "Score": 6.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.8, + // "ImpactScore": 3.6, + // "Score": 6.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-03-01T16:34Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1230,17 +1231,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2021-24112", @@ -1254,12 +1256,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1279,12 +1282,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:16Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1304,18 +1308,19 @@ var testCases = []testCase{ "Score": 4.6, "Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 1.3, - "ImpactScore": 5.9, - "Score": 7.3, - "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 1.3, + // "ImpactScore": 5.9, + // "Score": 7.3, + // "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T00:15Z", "PublishedDateTime": "2021-05-11T19:15Z", }, }, FixedBy: "3.1.15", - Severity: "Important", + Severity: "Moderate", }, }, }, @@ -1397,16 +1402,17 @@ var testCases = []testCase{ "ExploitabilityScore": 10.0, "ImpactScore": 2.9, }, - "CVSSv3": map[string]interface{}{ - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N", - "Score": 7.5, - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N", + // "Score": 7.5, + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // }, }, }, FixedBy: "3.1.8", - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2020-1161", @@ -1471,17 +1477,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, }, AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a", @@ -1605,12 +1612,13 @@ var testCases = []testCase{ "Score": 4.3, "Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.8, - "ImpactScore": 3.6, - "Score": 6.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.8, + // "ImpactScore": 3.6, + // "Score": 6.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-03-01T16:34Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1631,17 +1639,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2021-24112", @@ -1655,12 +1664,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1680,12 +1690,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1705,18 +1716,19 @@ var testCases = []testCase{ "Score": 4.6, "Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 1.3, - "ImpactScore": 5.9, - "Score": 7.3, - "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 1.3, + // "ImpactScore": 5.9, + // "Score": 7.3, + // "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T00:15Z", "PublishedDateTime": "2021-05-11T19:15Z", }, }, FixedBy: "3.1.15", - Severity: "Important", + Severity: "Moderate", }, }, AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a",