Skip to content

Commit

Permalink
cvss: switch tests to external fixtures
Browse files Browse the repository at this point in the history 
Putting all the fixtures into separate files makes adding new cases just
an `echo` instead of editing go source.

Signed-off-by: Hank Donnay <[email protected]>
  • Loading branch information
@hdonnay
hdonnay committed Dec 17, 2024
1 parent 4963daa commit c8eca51
Show file tree
Hide file tree
Showing 17 changed files with 360 additions and 325 deletions.
15 changes: 11 additions & 4 deletions toolkit/types/cvss/cvss_bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,23 @@ import (

// LoadVectorList loads a list of newline-separated CVSS vectors, omitting empty
// lines and lines starting with '#'.
//
// This is substantially the same as the "Load Fixture" functions, but only
// considers the first field and keeps the returned lines as []byte.
func loadVectorList(t testing.TB, name string) [][]byte {
t.Helper()
in, err := os.ReadFile(filepath.Join(`testdata`, name))
if err != nil {
t.Fatal(err)
}
vecs := bytes.Split(in, []byte{'\n'})
return slices.DeleteFunc(vecs, func(b []byte) bool {
vecs = slices.DeleteFunc(vecs, func(b []byte) bool {
return len(b) == 0 || b[0] == '#'
})
for i, vec := range vecs {
vecs[i] = bytes.Fields(vec)[0]
}
return vecs
}

func BenchmarkUnmarshal(b *testing.B) {
Expand All @@ -43,7 +50,7 @@ func benchmarkV4(b *testing.B) {

// Tests a list of different vectors.
b.Run("List", func(b *testing.B) {
vecs := loadVectorList(b, `v4_bench.list`)
vecs := loadVectorList(b, `v4_roundtrip.list`)
var n int64
for _, vec := range vecs {
n += int64(len(vec))
Expand Down Expand Up @@ -81,7 +88,7 @@ func benchmarkV4(b *testing.B) {
func benchmarkV3(b *testing.B) {
// Tests a list of different vectors.
b.Run("List", func(b *testing.B) {
vecs := loadVectorList(b, `v3_bench.list`)
vecs := loadVectorList(b, `v31_score.list`)
var n int64
for _, vec := range vecs {
n += int64(len(vec))
Expand Down Expand Up @@ -115,7 +122,7 @@ func benchmarkV3(b *testing.B) {
func benchmarkV2(b *testing.B) {
// Tests a list of different vectors.
b.Run("List", func(b *testing.B) {
vecs := loadVectorList(b, `v2_bench.list`)
vecs := loadVectorList(b, `v2_score.list`)
var n int64
for _, vec := range vecs {
n += int64(len(vec))
Expand Down
142 changes: 142 additions & 0 deletions toolkit/types/cvss/cvss_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package cvss

import (
"bufio"
"os"
"reflect"
"strconv"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
Expand Down Expand Up @@ -63,6 +67,41 @@ func Roundtrip[T any, M Metric, P VectorImpl[M, T]](t *testing.T, vecs []string)
}
}

// LoadRoundtripFixture loads the named file as a list of vectors.
//
// The format of the fixture is:
// - Vector string, one per line.
// - Empty lines and lines beginning with '#' are ignored.
// - Trailing words after a '#' are ignored.
func LoadRoundtripFixture(t testing.TB, file string) (vecs []string) {
t.Helper()
f, err := os.Open(file)
if err != nil {
t.Fatalf("opening roundtrip fixture %q: %v", file, err)
}
s := bufio.NewScanner(f)
defer func() {
if err := s.Err(); err != nil {
t.Errorf("bufio error: %v", err)
}
if err := f.Close(); err != nil {
t.Errorf("close error: %v", err)
}
}()

for s.Scan() {
l := s.Text()
if len(l) == 0 || l[0] == '#' {
continue
}
if idx := strings.IndexByte(l, '#'); idx != -1 {
l = strings.TrimSpace(l[:idx])
}
vecs = append(vecs, l)
}
return vecs
}

type ScoreTestcase struct {
Vector string
Score float64
Expand Down Expand Up @@ -101,6 +140,57 @@ func Score[T any, M Metric, P VectorImpl[M, T]](t *testing.T, tcs []ScoreTestcas
}
}

// LoadScoreFixture loads the name file as a list of [ScoreTestcase].
//
// The format of the fixture is:
// - Space separated vector and decimal number, one per line.
// - Empty lines and lines beginning with '#' are ignored.
// - Fields beyond the first two are ignored.
func LoadScoreFixture(t testing.TB, file string) (tcs []ScoreTestcase) {
t.Helper()
f, err := os.Open(file)
if err != nil {
t.Fatalf("opening score fixture %q: %v", file, err)
}
s := bufio.NewScanner(f)
defer func() {
if err := s.Err(); err != nil {
t.Errorf("bufio error: %v", err)
}
if err := f.Close(); err != nil {
t.Errorf("close error: %v", err)
}
}()

Line:
for s.Scan() {
l := s.Text()
if len(l) == 0 || l[0] == '#' {
continue
}
fs := strings.Fields(l)
var tc ScoreTestcase
Field:
for i, f := range fs {
switch i {
case 0:
tc.Vector = f
case 1:
s, err := strconv.ParseFloat(f, 64)
if err != nil {
t.Errorf("bad float (%v), skipping: %#q", err, l)
continue Line
}
tc.Score = s
default:
break Field
}
}
tcs = append(tcs, tc)
}
return tcs
}

type ErrorTestcase struct {
Vector string
Error bool
Expand All @@ -121,3 +211,55 @@ func Error[T any, M Metric, P VectorImpl[M, T]](t *testing.T, tcs []ErrorTestcas
})
}
}

// LoadErrorFixture loads the named file as a list of vectors.
//
// The format of the fixture is:
// - Vector string, one per line.
// - Empty lines and lines beginning with '#' are ignored.
// - A trailing word with "#OK" marks the line as _not_ failing.
func LoadErrorFixture(t testing.TB, file string) (tcs []ErrorTestcase) {
t.Helper()
f, err := os.Open(file)
if err != nil {
t.Fatalf("opening error fixture %q: %v", file, err)
}
s := bufio.NewScanner(f)
defer func() {
if err := s.Err(); err != nil {
t.Errorf("bufio error: %v", err)
}
if err := f.Close(); err != nil {
t.Errorf("close error: %v", err)
}
}()

Line:
for s.Scan() {
l := s.Text()
if len(l) == 0 || l[0] == '#' {
continue
}
tc := ErrorTestcase{
Error: true,
}
fs := strings.Fields(l)
for i, f := range fs {
switch i {
case 0:
tc.Vector = f
case 1:
if f == `#OK` {
tc.Error = false
break
}
fallthrough
default:
t.Errorf("odd line, skipping: %#q", l)
continue Line
}
}
tcs = append(tcs, tc)
}
return tcs
}
42 changes: 3 additions & 39 deletions toolkit/types/cvss/cvss_v2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,49 +7,13 @@ import (

func TestV2(t *testing.T) {
t.Run("Error", func(t *testing.T) {
tcs := []ErrorTestcase{
{Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:C"},
{Vector: "AV:N/AC:L/Au:N/C:C/I:C/A:C"},
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C"},
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:C/CDP:H/TD:H/CR:M/IR:M/AR:H"},
{Vector: "CVSS:2.0/AV:N/AC:L/Au:N/C:N/I:N/A:C", Error: true},
{Vector: "AV:N/AC:L/Au:N/C:N/I:N", Error: true},
{Vector: "AV:A/AC:L/Au:N/C:C/I:C/A:C/CDP:H/TD:H/CR:H", Error: true},
{Vector: "AV:A/AC:L/Au:N/C:C/I:C/A:C/E:F", Error: true},
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C/Au:N", Error: true},
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:", Error: true},
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C?notaurl=1", Error: true},
{Vector: "AV:A/AC:L/Au:N/A:C/I:C/C:C", Error: true},
}
Error[V2, V2Metric, *V2](t, tcs)
Error[V2, V2Metric, *V2](t, LoadErrorFixture(t, "testdata/v2_error.list"))
})
t.Run("Roundtrip", func(t *testing.T) {
vecs := []string{
"AV:N/AC:L/Au:N/C:N/I:N/A:C", // CVE-2002-0392
"AV:N/AC:L/Au:N/C:C/I:C/A:C", // CVE-2003-0818
"AV:L/AC:H/Au:N/C:C/I:C/A:C", // CVE-2003-0062
"AV:L/AC:H/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:C/CDP:H/TD:H/CR:M/IR:M/AR:H", // CVE-2002-0392
"AV:L/AC:H/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:UR/CDP:ND/TD:ND/CR:ND/IR:ND/AR:ND", // made up
"AV:L/AC:H/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:UR/CDP:LM/TD:ND/CR:ND/IR:ND/AR:ND", // made up
}
Roundtrip[V2, V2Metric, *V2](t, vecs)
Roundtrip[V2, V2Metric, *V2](t, LoadRoundtripFixture(t, "testdata/v2_roundtrip.list"))
})
t.Run("Score", func(t *testing.T) {
tcs := []ScoreTestcase{
{Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:C", Score: 7.8}, // CVE-2002-0392
{Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:C/E:F/RL:OF/RC:C", Score: 6.4}, // CVE-2002-0392
{Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:C/E:F/RL:OF/RC:C/CDP:N/TD:N/CR:M/IR:M/AR:H", Score: 0.0}, // CVE-2002-0392
{Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:C/E:F/RL:OF/RC:C/CDP:H/TD:H/CR:M/IR:M/AR:H", Score: 9.2}, // CVE-2002-0392
{Vector: "AV:N/AC:L/Au:N/C:C/I:C/A:C", Score: 10.0}, // CVE-2003-0818
{Vector: "AV:N/AC:L/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:C", Score: 8.3}, // CVE-2003-0818
{Vector: "AV:N/AC:L/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:C/CDP:N/TD:N/CR:M/IR:M/AR:L", Score: 0.0}, // CVE-2003-0818
{Vector: "AV:N/AC:L/Au:N/C:C/I:C/A:C/E:F/RL:OF/RC:C/CDP:H/TD:H/CR:M/IR:M/AR:L", Score: 9.0}, // CVE-2003-0818
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C", Score: 6.2}, // CVE-2003-0062
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C/E:POC/RL:OF/RC:C", Score: 4.9}, // CVE-2003-0062
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C/E:POC/RL:OF/RC:C/CDP:N/TD:N/CR:M/IR:M/AR:M", Score: 0.0}, // CVE-2003-0062
{Vector: "AV:L/AC:H/Au:N/C:C/I:C/A:C/E:POC/RL:OF/RC:C/CDP:H/TD:H/CR:M/IR:M/AR:M", Score: 7.5}, // CVE-2003-0062
}
Score[V2, V2Metric, *V2](t, tcs)
Score[V2, V2Metric, *V2](t, LoadScoreFixture(t, "testdata/v2_score.list"))
})

t.Run("Unparse", func(t *testing.T) {
Expand Down
Loading

0 comments on commit c8eca51

Please sign in to comment.