From 73b0613b3ac5fa946e3f62c87d9286a37bbc40c3 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Mon, 9 Dec 2024 20:35:09 -0600 Subject: [PATCH 1/5] adjusts test cases and logic to handle removed NVD data --- e2etests/sanity_test.go | 27 ++++-- e2etests/testcase_test.go | 168 ++++++++++++++++++++------------------ 2 files changed, 109 insertions(+), 86 deletions(-) diff --git a/e2etests/sanity_test.go b/e2etests/sanity_test.go index e05823091..7eddfefce 100644 --- a/e2etests/sanity_test.go +++ b/e2etests/sanity_test.go @@ -20,18 +20,29 @@ func checkMatch(t *testing.T, source string, expectedVuln, matchingVuln v1.Vulne if expectedVuln.Metadata == nil { assert.Nil(t, matchingVuln.Metadata, "Expected no metadata for %s but got some", expectedVuln.Name) } else { - for _, keys := range [][]string{ + keySets := [][]string{ {source, "CVSSv2", "ExploitabilityScore"}, {source, "CVSSv2", "Score"}, {source, "CVSSv2", "ImpactScore"}, {source, "CVSSv2", "Vectors"}, - {source, "CVSSv3", "ExploitabilityScore"}, - {source, "CVSSv3", "Score"}, - {source, "CVSSv3", "ImpactScore"}, - {source, "CVSSv3", "Vectors"}, - } { - assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v", keys) - assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v", keys) + } + + // When expected vuln has no CVSSv3 data, do not try to compare it. + // This was added when NVD stopped returning CVSSv3 data for some vulns which we had test cases for. + if deepGet(expectedVuln.Metadata, []string{source, "CVSSv3"}...) != nil { + keySets = append(keySets, [][]string{ + {source, "CVSSv3", "ExploitabilityScore"}, + {source, "CVSSv3", "Score"}, + {source, "CVSSv3", "ImpactScore"}, + {source, "CVSSv3", "Vectors"}, + }...) + } else { + t.Logf("WARN: No CVSSv3 data provided for %q, skipping CVSSv3 field validation.", expectedVuln.Name) + } + + for _, keys := range keySets { + assert.NotNil(t, deepGet(expectedVuln.Metadata, keys...), "Value for nil for %+v in vuln %q", keys, expectedVuln.Name) + assert.Equal(t, deepGet(expectedVuln.Metadata, keys...), deepGet(matchingVuln.Metadata, keys...), "Failed for %+v in vuln %q", keys, expectedVuln.Name) } } expectedVuln.Metadata = nil diff --git a/e2etests/testcase_test.go b/e2etests/testcase_test.go index 323752bf9..5a9cb18e0 100644 --- a/e2etests/testcase_test.go +++ b/e2etests/testcase_test.go @@ -1204,12 +1204,13 @@ var testCases = []testCase{ "Score": 4.3, "Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.8, - "ImpactScore": 3.6, - "Score": 6.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.8, + // "ImpactScore": 3.6, + // "Score": 6.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-03-01T16:34Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1230,17 +1231,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2021-24112", @@ -1254,12 +1256,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1279,12 +1282,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:16Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1304,18 +1308,19 @@ var testCases = []testCase{ "Score": 4.6, "Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 1.3, - "ImpactScore": 5.9, - "Score": 7.3, - "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 1.3, + // "ImpactScore": 5.9, + // "Score": 7.3, + // "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T00:15Z", "PublishedDateTime": "2021-05-11T19:15Z", }, }, FixedBy: "3.1.15", - Severity: "Important", + Severity: "Moderate", }, }, }, @@ -1397,16 +1402,17 @@ var testCases = []testCase{ "ExploitabilityScore": 10.0, "ImpactScore": 2.9, }, - "CVSSv3": map[string]interface{}{ - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N", - "Score": 7.5, - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N", + // "Score": 7.5, + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // }, }, }, FixedBy: "3.1.8", - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2020-1161", @@ -1471,17 +1477,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, }, AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a", @@ -1605,12 +1612,13 @@ var testCases = []testCase{ "Score": 4.3, "Vectors": "AV:N/AC:M/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.8, - "ImpactScore": 3.6, - "Score": 6.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.8, + // "ImpactScore": 3.6, + // "Score": 6.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-03-01T16:34Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1631,17 +1639,18 @@ var testCases = []testCase{ "Score": 5.0, "Vectors": "AV:N/AC:L/Au:N/C:N/I:N/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 3.9, - "ImpactScore": 3.6, - "Score": 7.5, - "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 3.9, + // "ImpactScore": 3.6, + // "Score": 7.5, + // "Vectors": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + // }, "LastModifiedDateTime": "2021-01-25T19:54Z", "PublishedDateTime": "2021-01-12T20:15Z", }, }, - Severity: "Important", + Severity: "Moderate", }, { Name: "CVE-2021-24112", @@ -1655,12 +1664,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1680,12 +1690,13 @@ var testCases = []testCase{ "Score": 7.5, "Vectors": "AV:N/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.9, - "Score": 8.1, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 2.2, + // "ImpactScore": 5.9, + // "Score": 8.1, + // "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T17:15Z", "PublishedDateTime": "2021-02-25T23:15Z", }, @@ -1705,18 +1716,19 @@ var testCases = []testCase{ "Score": 4.6, "Vectors": "AV:L/AC:L/Au:N/C:P/I:P/A:P", }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 1.3, - "ImpactScore": 5.9, - "Score": 7.3, - "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", - }, + // NVD stopped returning this + // "CVSSv3": map[string]interface{}{ + // "ExploitabilityScore": 1.3, + // "ImpactScore": 5.9, + // "Score": 7.3, + // "Vectors": "CVSS:3.1/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", + // }, "LastModifiedDateTime": "2023-12-29T00:15Z", "PublishedDateTime": "2021-05-11T19:15Z", }, }, FixedBy: "3.1.15", - Severity: "Important", + Severity: "Moderate", }, }, AddedBy: "sha256:5bd47e7e8ad7786db14c79827b543615728f0e27567f5b05d4c13db29bb24c7a", From e7d0542aedb2ecbbd7538ccde82ec47b656cc3e3 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:22:24 -0600 Subject: [PATCH 2/5] increase test timeouts --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6326b509a..7cb8fcaf8 100644 --- a/Makefile +++ b/Makefile @@ -330,13 +330,13 @@ unit-tests: deps test-prep e2e-tests: deps test-prep @echo "+ $@" set -o pipefail ; \ - go test -tags e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log + go test -tags e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log .PHONY: slim-e2e-tests slim-e2e-tests: deps test-prep @echo "+ $@" set -o pipefail ; \ - go test -tags slim_e2e -count=1 -timeout=20m -v ./e2etests/... | tee test-output/test.log + go test -tags slim_e2e -count=1 -timeout=30m -v ./e2etests/... | tee test-output/test.log .PHONY: db-integration-tests db-integration-tests: deps test-prep From 85cdbb070b7c10bfec690234441e66f01d5884b4 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:42:28 -0600 Subject: [PATCH 3/5] fix jenkins test case --- e2etests/testcase_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/e2etests/testcase_test.go b/e2etests/testcase_test.go index 5a9cb18e0..9a508e1dc 100644 --- a/e2etests/testcase_test.go +++ b/e2etests/testcase_test.go @@ -3899,31 +3899,6 @@ Applications using RegexRequestMatcher with '.' in the regular expression are po AddedBy: "sha256:3fa3f612bdcb92746bf76be1b9c9e1c1c80de777aedaf48b7068f4a129ded3c2", FixedBy: "4.10.1685679861-1.el8", Vulnerabilities: []apiV1.Vulnerability{ - { - Name: "CVE-2021-26291", - NamespaceName: "rhel:8", - Description: `DOCUMENTATION: A flaw was found in maven. Repositories that are defined in a dependency’s Project Object Model (pom), which may be unknown to users, are used by default resulting in potential risk if a malicious actor takes over that repository or is able to insert themselves into a position to pretend to be that repository. The highest threat from this vulnerability is to data confidentiality and integrity. - - MITIGATION: To avoid possible man-in-the-middle related attacks with this flaw, ensure any linked repositories in maven POMs use https and not http.`, - Link: "https://access.redhat.com/security/cve/CVE-2021-26291", - Severity: "Moderate", - Metadata: map[string]interface{}{ - "Red Hat": map[string]interface{}{ - "CVSSv2": map[string]interface{}{ - "ExploitabilityScore": 0.0, - "ImpactScore": 0.0, - "Score": 0.0, - "Vectors": "", - }, - "CVSSv3": map[string]interface{}{ - "ExploitabilityScore": 2.2, - "ImpactScore": 5.2, - "Score": 7.4, - "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:N", - }, - }, - }, - }, { Name: "RHSA-2022:6531", NamespaceName: "rhel:8", From 408224cdb46f476c0e704bd72a52d93e706e7f7e Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:30:32 -0600 Subject: [PATCH 4/5] fail test if v3 data exists but not tested --- e2etests/sanity_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/e2etests/sanity_test.go b/e2etests/sanity_test.go index 7eddfefce..d9c1c3106 100644 --- a/e2etests/sanity_test.go +++ b/e2etests/sanity_test.go @@ -27,9 +27,20 @@ func checkMatch(t *testing.T, source string, expectedVuln, matchingVuln v1.Vulne {source, "CVSSv2", "Vectors"}, } - // When expected vuln has no CVSSv3 data, do not try to compare it. - // This was added when NVD stopped returning CVSSv3 data for some vulns which we had test cases for. - if deepGet(expectedVuln.Metadata, []string{source, "CVSSv3"}...) != nil { + testHasCVSSv3 := deepGet(expectedVuln.Metadata, []string{source, "CVSSv3"}...) != nil + + vulnCVSSv3 := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3"}...) + vulnCVSSv3Vectors := deepGet(matchingVuln.Metadata, []string{source, "CVSSv3", "Vectors"}...) + // Assume that CVSSv3 scores exist if Vectors is not empty. + vulnHasCVSSv3 := vulnCVSSv3Vectors != nil && vulnCVSSv3Vectors.(string) != "" + + // If the test case does not have CVSSv3 scores but the matching vuln does, fail the test. + // This was added when NVD stopped returning CVSSv3 data for some vulns which we had + // test cases for, this condition ensures we are alerted when/if the data returns. + assert.False(t, !testHasCVSSv3 && vulnHasCVSSv3, "Test case for %q is missing CVSSv3 scores, please add the scores to the test case. Scores from vuln data: %+v", expectedVuln.Name, vulnCVSSv3) + + // Compare CVSSv3 data only when it exists in the test case. + if testHasCVSSv3 { keySets = append(keySets, [][]string{ {source, "CVSSv3", "ExploitabilityScore"}, {source, "CVSSv3", "Score"}, From a2e94b7128630d1bca71ff3e3339b89825450192 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:19:57 -0600 Subject: [PATCH 5/5] put flakey test back --- e2etests/testcase_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/e2etests/testcase_test.go b/e2etests/testcase_test.go index 9a508e1dc..5a9cb18e0 100644 --- a/e2etests/testcase_test.go +++ b/e2etests/testcase_test.go @@ -3899,6 +3899,31 @@ Applications using RegexRequestMatcher with '.' in the regular expression are po AddedBy: "sha256:3fa3f612bdcb92746bf76be1b9c9e1c1c80de777aedaf48b7068f4a129ded3c2", FixedBy: "4.10.1685679861-1.el8", Vulnerabilities: []apiV1.Vulnerability{ + { + Name: "CVE-2021-26291", + NamespaceName: "rhel:8", + Description: `DOCUMENTATION: A flaw was found in maven. Repositories that are defined in a dependency’s Project Object Model (pom), which may be unknown to users, are used by default resulting in potential risk if a malicious actor takes over that repository or is able to insert themselves into a position to pretend to be that repository. The highest threat from this vulnerability is to data confidentiality and integrity. + + MITIGATION: To avoid possible man-in-the-middle related attacks with this flaw, ensure any linked repositories in maven POMs use https and not http.`, + Link: "https://access.redhat.com/security/cve/CVE-2021-26291", + Severity: "Moderate", + Metadata: map[string]interface{}{ + "Red Hat": map[string]interface{}{ + "CVSSv2": map[string]interface{}{ + "ExploitabilityScore": 0.0, + "ImpactScore": 0.0, + "Score": 0.0, + "Vectors": "", + }, + "CVSSv3": map[string]interface{}{ + "ExploitabilityScore": 2.2, + "ImpactScore": 5.2, + "Score": 7.4, + "Vectors": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:N", + }, + }, + }, + }, { Name: "RHSA-2022:6531", NamespaceName: "rhel:8",