Skip to content

Commit

Permalink
fix(ci): NVD data and test issues (#1737)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcaravel
dcaravel authored Dec 19, 2024
1 parent fb6e36e commit 71cbd4b
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 88 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,13 +330,13 @@ unit-tests: deps test-prep
e2e-tests: deps test-prep
@echo "+ $@"
set -o pipefail ; \
go test -tags e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log
go test -tags e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log

.PHONY: slim-e2e-tests
slim-e2e-tests: deps test-prep
@echo "+ $@"
set -o pipefail ; \
go test -tags slim_e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log
go test -tags slim_e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log

.PHONY: db-integration-tests
db-integration-tests: deps test-prep
Expand Down
38 changes: 30 additions & 8 deletions e2etests/sanity_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,40 @@ func checkMatch(t *testing.T, source string, expectedVuln, matchingVuln v1.Vulne
if expectedVuln.Metadata == nil {
assert.Nil(t, matchingVuln.Metadata, "Expected no metadata for %s but got some", expectedVuln.Name)
} else {
for _, keys := range [][]string{
keySets := [][]string{
{source, "CVSSv2", "ExploitabilityScore"},
{source, "CVSSv2", "Score"},
{source, "CVSSv2", "ImpactScore"},
{source, "CVSSv2", "Vectors"},
{source, "CVSSv3", "ExploitabilityScore"},
{source, "CVSSv3", "Score"},
{source, "CVSSv3", "ImpactScore"},
{source, "CVSSv3", "Vectors"},
} {
assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v", keys)
assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v", keys)
}

testHasCVSSv3 := deepGet(expectedVuln.Metadata, []string{source, "CVSSv3"}...) != nil

vulnCVSSv3 := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3"}...)
vulnCVSSv3Vectors := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3", "Vectors"}...)
// Assume that CVSSv3 scores exist if Vectors is not empty.
vulnHasCVSSv3 := vulnCVSSv3Vectors != nil && vulnCVSSv3Vectors.(string) != ""

// If the test case does not have CVSSv3 scores but the matching vuln does, fail the test.
// This was added when NVD stopped returning CVSSv3 data for some vulns which we had
// test cases for, this condition ensures we are alerted when/if the data returns.
assert.False(t, !testHasCVSSv3 && vulnHasCVSSv3, "Test case for %q is missing CVSSv3 scores, please add the scores to the test case. Scores from vuln data: %+v", expectedVuln.Name, vulnCVSSv3)

// Compare CVSSv3 data only when it exists in the test case.
if testHasCVSSv3 {
keySets = append(keySets, [][]string{
{source, "CVSSv3", "ExploitabilityScore"},
{source, "CVSSv3", "Score"},
{source, "CVSSv3", "ImpactScore"},
{source, "CVSSv3", "Vectors"},
}...)
} else {
t.Logf("WARN: No CVSSv3 data provided for %q, skipping CVSSv3 field validation.", expectedVuln.Name)
}

for _, keys := range keySets {
assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v in vuln %q", keys, expectedVuln.Name)
assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v in vuln %q", keys, expectedVuln.Name)
}
}
expectedVuln.Metadata = nil
Expand Down
168 changes: 90 additions & 78 deletions e2etests/testcase_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1204,12 +1204,13 @@ var testCases = []testCase{
"Score": 4.3,
"Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.8,
"ImpactScore": 3.6,
"Score": 6.5,
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.8,
// "ImpactScore": 3.6,
// "Score": 6.5,
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H",
// },
"LastModifiedDateTime": "2021-03-01T16:34Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1230,17 +1231,18 @@ var testCases = []testCase{
"Score": 5.0,
"Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 3.9,
"ImpactScore": 3.6,
"Score": 7.5,
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 3.9,
// "ImpactScore": 3.6,
// "Score": 7.5,
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
// },
"LastModifiedDateTime": "2021-01-25T19:54Z",
"PublishedDateTime": "2021-01-12T20:15Z",
},
},
Severity: "Important",
Severity: "Moderate",
},
{
Name: "CVE-2021-24112",
Expand All @@ -1254,12 +1256,13 @@ var testCases = []testCase{
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.2,
"ImpactScore": 5.9,
"Score": 8.1,
"Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.2,
// "ImpactScore": 5.9,
// "Score": 8.1,
// "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T17:15Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1279,12 +1282,13 @@ var testCases = []testCase{
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.2,
"ImpactScore": 5.9,
"Score": 8.1,
"Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.2,
// "ImpactScore": 5.9,
// "Score": 8.1,
// "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T17:16Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1304,18 +1308,19 @@ var testCases = []testCase{
"Score": 4.6,
"Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 1.3,
"ImpactScore": 5.9,
"Score": 7.3,
"Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 1.3,
// "ImpactScore": 5.9,
// "Score": 7.3,
// "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T00:15Z",
"PublishedDateTime": "2021-05-11T19:15Z",
},
},
FixedBy: "3.1.15",
Severity: "Important",
Severity: "Moderate",
},
},
},
Expand Down Expand Up @@ -1397,16 +1402,17 @@ var testCases = []testCase{
"ExploitabilityScore": 10.0,
"ImpactScore": 2.9,
},
"CVSSv3": map[string]interface{}{
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N",
"Score": 7.5,
"ExploitabilityScore": 3.9,
"ImpactScore": 3.6,
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N",
// "Score": 7.5,
// "ExploitabilityScore": 3.9,
// "ImpactScore": 3.6,
// },
},
},
FixedBy: "3.1.8",
Severity: "Important",
Severity: "Moderate",
},
{
Name: "CVE-2020-1161",
Expand Down Expand Up @@ -1471,17 +1477,18 @@ var testCases = []testCase{
"Score": 5.0,
"Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 3.9,
"ImpactScore": 3.6,
"Score": 7.5,
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 3.9,
// "ImpactScore": 3.6,
// "Score": 7.5,
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
// },
"LastModifiedDateTime": "2021-01-25T19:54Z",
"PublishedDateTime": "2021-01-12T20:15Z",
},
},
Severity: "Important",
Severity: "Moderate",
},
},
AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a",
Expand Down Expand Up @@ -1605,12 +1612,13 @@ var testCases = []testCase{
"Score": 4.3,
"Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.8,
"ImpactScore": 3.6,
"Score": 6.5,
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.8,
// "ImpactScore": 3.6,
// "Score": 6.5,
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H",
// },
"LastModifiedDateTime": "2021-03-01T16:34Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1631,17 +1639,18 @@ var testCases = []testCase{
"Score": 5.0,
"Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 3.9,
"ImpactScore": 3.6,
"Score": 7.5,
"Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 3.9,
// "ImpactScore": 3.6,
// "Score": 7.5,
// "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
// },
"LastModifiedDateTime": "2021-01-25T19:54Z",
"PublishedDateTime": "2021-01-12T20:15Z",
},
},
Severity: "Important",
Severity: "Moderate",
},
{
Name: "CVE-2021-24112",
Expand All @@ -1655,12 +1664,13 @@ var testCases = []testCase{
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.2,
"ImpactScore": 5.9,
"Score": 8.1,
"Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.2,
// "ImpactScore": 5.9,
// "Score": 8.1,
// "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T17:15Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1680,12 +1690,13 @@ var testCases = []testCase{
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 2.2,
"ImpactScore": 5.9,
"Score": 8.1,
"Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 2.2,
// "ImpactScore": 5.9,
// "Score": 8.1,
// "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T17:15Z",
"PublishedDateTime": "2021-02-25T23:15Z",
},
Expand All @@ -1705,18 +1716,19 @@ var testCases = []testCase{
"Score": 4.6,
"Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P",
},
"CVSSv3": map[string]interface{}{
"ExploitabilityScore": 1.3,
"ImpactScore": 5.9,
"Score": 7.3,
"Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H",
},
// NVD stopped returning this
// "CVSSv3": map[string]interface{}{
// "ExploitabilityScore": 1.3,
// "ImpactScore": 5.9,
// "Score": 7.3,
// "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H",
// },
"LastModifiedDateTime": "2023-12-29T00:15Z",
"PublishedDateTime": "2021-05-11T19:15Z",
},
},
FixedBy: "3.1.15",
Severity: "Important",
Severity: "Moderate",
},
},
AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a",
Expand Down

0 comments on commit 71cbd4b

Please sign in to comment.