From 1545839203a882db564a37bd9c784e81cc203052 Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Mon, 2 Jan 2023 07:20:57 +0100 Subject: [PATCH 1/6] Add .idea to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6589552..e4145c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ junit2jira +.idea From bf1ce87c090e16badee854e36cb40e256d3ea3f1 Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Mon, 2 Jan 2023 07:22:41 +0100 Subject: [PATCH 2/6] Group sub-tests based on delimiters; add Errors field to avoid only having Failed in ticket's message. --- main.go | 51 +++++++- main_test.go | 62 +++++----- testdata/report.xml | 281 +++++++++++++++++++++++--------------------- 3 files changed, 225 insertions(+), 169 deletions(-) diff --git a/main.go b/main.go index f5621d4..bb6799b 100644 --- a/main.go +++ b/main.go @@ -186,7 +186,7 @@ func findFailedTests(dirName string, env map[string]string, threshold int) ([]te if tc.Error == nil { continue } - failedTests = append(failedTests, NewTestCase(tc, env)) + failedTests = addFailedTest(failedTests, tc, env) } } log.Printf("Found %d failed tests", len(failedTests)) @@ -220,6 +220,32 @@ func mergeFailedTests(failedTests []testCase, env map[string]string) ([]testCase return []testCase{tc}, nil } +func addFailedTest(failedTests []testCase, tc junit.Test, env map[string]string) []testCase { + if !isSubTest(tc) { + return append(failedTests, NewTestCase(tc, env)) + } + return addSubTestToFailedTest(tc, failedTests, env) +} + +func isSubTest(tc junit.Test) bool { + return strings.Contains(tc.Name, "/") +} + +func addSubTestToFailedTest(subTest junit.Test, failedTests []testCase, env map[string]string) []testCase { + // As long as the separator is not empty, split will always return a slice of length 1. + name := strings.Split(subTest.Name, "/")[0] + for i, failedTest := range failedTests { + // Only consider a failed test a "parent" of the test if the name matches _and_ the class name is the same. + if failedTest.Name == name && failedTest.Suite == subTest.Classname { + failedTest.addSubTest(subTest) + failedTests[i] = failedTest + return failedTests + } + } + // In case we found no matches, we will default to add the subtest plain. + return append(failedTests, NewTestCase(subTest, env)) +} + const ( desc = ` {{- if .Message }} @@ -254,6 +280,7 @@ type testCase struct { Message string Stdout string Stderr string + Error string BuildId string Cluster string JobName string @@ -270,6 +297,7 @@ func NewTestCase(tc junit.Test, env map[string]string) testCase { Message: tc.Message, Stdout: tc.SystemOut, Stderr: tc.SystemErr, + Error: tc.Error.Error(), Suite: tc.Classname, BuildId: env["BUILD_ID"], Cluster: env["CLUSTER_NAME"], @@ -280,8 +308,8 @@ func NewTestCase(tc junit.Test, env map[string]string) testCase { } } -func (tc testCase) description() (string, error) { - return render(tc, desc) +func (tc *testCase) description() (string, error) { + return render(*tc, desc) } func (tc testCase) summary() (string, error) { @@ -292,6 +320,23 @@ func (tc testCase) summary() (string, error) { return clearString(s), nil } +const subTestFormat = "\nSub test %s: %s" + +func (tc *testCase) addSubTest(subTest junit.Test) { + if subTest.Message != "" { + tc.Message += fmt.Sprintf(subTestFormat, subTest.Name, subTest.Message) + } + if subTest.SystemOut != "" { + tc.Stdout += fmt.Sprintf(subTestFormat, subTest.Name, subTest.SystemOut) + } + if subTest.SystemErr != "" { + tc.Stderr += fmt.Sprintf(subTestFormat, subTest.Name, subTest.SystemErr) + } + if subTest.Error != nil { + tc.Error += fmt.Sprintf(subTestFormat, subTest.Name, subTest.Error.Error()) + } +} + func render(tc testCase, text string) (string, error) { tmpl, err := template.New("test").Parse(text) if err != nil { diff --git a/main_test.go b/main_test.go index 9392f85..c341127 100644 --- a/main_test.go +++ b/main_test.go @@ -16,24 +16,17 @@ func TestParseJunitReport(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []testCase{ { - Name: "TestLocalScannerTLSIssuerIntegrationTests", - Message: "Failed", - Stdout: "", - Stderr: "", - Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", - }, - { - Name: "TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh", - Message: "Failed", - Stdout: "", - Stderr: "", - Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", + Name: "TestDifferentBaseTypes", + Suite: "github.com/stackrox/rox/pkg/booleanpolicy/evaluator", + Message: "Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object: Failed", + Error: "Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object: \n evaluator_test.go:96: Error Trace: /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:96 /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:123 Error: Not equal: expected: false actual : true Test: TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object \n \nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object: \n evaluator_test.go:96: Error Trace: /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:96 /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:145 Error: Not equal: expected: false actual : true Test: TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object \n ", }, { - Name: "TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets", - Message: "Failed", + Name: "TestLocalScannerTLSIssuerIntegrationTests", + Message: "Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh: Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets: Failed", Stdout: "", Stderr: "", + Error: " env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CERT_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-cert.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CERT_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-cert.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-key.pem\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh: Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets: tls_issuer_test.go:377:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/tls_issuer_test.go:377\n \t \t\t\t\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/tls_issuer_test.go:298\n \t \t\t\t\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/suite.go:91\n \tError: \tcontext deadline exceeded\n \tTest: \tTestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets\nkubernetes/localscanner: 2022/10/03 07:32:47.446934 cert_refresher.go:109: Warn: local scanner certificates not found (this is expected on a new deployment), will refresh certificates immediately: 2 errors occurred:\n\t* secrets \"scanner-tls\" not found\n\t* secrets \"scanner-db-tls\" not found\n\n", Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", }, }, tests) @@ -107,35 +100,33 @@ github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres / Te "Exit: 0\n", Suite: "DefaultPoliciesTest", BuildId: "1", + Error: "Condition not satisfied:\n" + + "\n" + + "waitForViolation(deploymentName, policyName, 60)\n" + + "| | |\n" + + "false qadefpolstruts Apache Struts: CVE-2017-5638\n" + + "\n" + + "\tat DefaultPoliciesTest.Verify policy #policyName is triggered(DefaultPoliciesTest.groovy:181)\n", }, { - Name: "TestLocalScannerTLSIssuerIntegrationTests", - Message: "Failed", - Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", - BuildId: "1", - }, - { - Name: "TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh", - Message: "Failed", - Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", + Name: "TestDifferentBaseTypes", + Suite: "github.com/stackrox/rox/pkg/booleanpolicy/evaluator", + Message: "Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object: Failed", + Error: "Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match: Failed\nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object: \n evaluator_test.go:96: Error Trace: /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:96 /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:123 Error: Not equal: expected: false actual : true Test: TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_fully_hydrated_object \n \nSub test TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object: \n evaluator_test.go:96: Error Trace: /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:96 /go/src/github.com/stackrox/stackrox/pkg/booleanpolicy/evaluator/evaluator_test.go:145 Error: Not equal: expected: false actual : true Test: TestDifferentBaseTypes/base_ts,_query_by_relative,_does_not_match/on_augmented_object \n ", BuildId: "1", }, { - Name: "TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets", - Message: "Failed", + Name: "TestLocalScannerTLSIssuerIntegrationTests", + Message: "Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh: Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets: Failed", Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", + Error: " env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CERT_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-cert.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CA_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/ca-key.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_CERT_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-cert.pem\n env_isolator.go:41: EnvIsolator: Setting ROX_MTLS_KEY_FILE to /go/src/github.com/stackrox/stackrox/pkg/mtls/testutils/testdata/central-certs/leaf-key.pem\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh: Failed\nSub test TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets: tls_issuer_test.go:377:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/tls_issuer_test.go:377\n \t \t\t\t\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/tls_issuer_test.go:298\n \t \t\t\t\t/go/src/github.com/stackrox/stackrox/sensor/kubernetes/localscanner/suite.go:91\n \tError: \tcontext deadline exceeded\n \tTest: \tTestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets\nkubernetes/localscanner: 2022/10/03 07:32:47.446934 cert_refresher.go:109: Warn: local scanner certificates not found (this is expected on a new deployment), will refresh certificates immediately: 2 errors occurred:\n\t* secrets \"scanner-tls\" not found\n\t* secrets \"scanner-db-tls\" not found\n\n", BuildId: "1", }, { Name: "TestCollectionsStore", Suite: "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres", - Message: "Failed", - BuildId: "1", - }, - { - Name: "TestCollectionsStore/TestStore", - Suite: "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres", - Message: "Failed", + Message: "Failed\nSub test TestCollectionsStore/TestStore: Failed", + Error: " env_isolator.go:41: EnvIsolator: Setting ROX_POSTGRES_DATASTORE to true\nSub test TestCollectionsStore/TestStore: store_test.go:47: collections TRUNCATE TABLE\n store_test.go:95:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store_test.go:95\n \tError: \tReceived unexpected error:\n \t \tERROR: update or delete on table \"collections\" violates foreign key constraint \"fk_collections_embedded_collections_collections_cycle_ref\" on table \"collections_embedded_collections\" (SQLSTATE 23503)\n \t \tcould not delete from \"collections\"\n \t \tgithub.com/stackrox/rox/pkg/search/postgres.RunDeleteRequestForSchema.func1\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/search/postgres/common.go:833\n \t \tgithub.com/stackrox/rox/pkg/postgres/pgutils.Retry.func1\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/postgres/pgutils/retry.go:21\n \t \tgithub.com/stackrox/rox/pkg/postgres/pgutils.Retry2[...].func1\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/postgres/pgutils/retry.go:32\n \t \tgithub.com/stackrox/rox/pkg/postgres/pgutils.Retry3[...]\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/postgres/pgutils/retry.go:43\n \t \tgithub.com/stackrox/rox/pkg/postgres/pgutils.Retry2[...]\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/postgres/pgutils/retry.go:35\n \t \tgithub.com/stackrox/rox/pkg/postgres/pgutils.Retry\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/postgres/pgutils/retry.go:23\n \t \tgithub.com/stackrox/rox/pkg/search/postgres.RunDeleteRequestForSchema\n \t \t\t/go/src/github.com/stackrox/stackrox/pkg/search/postgres/common.go:830\n \t \tgithub.com/stackrox/rox/central/resourcecollection/datastore/store/postgres.(*storeImpl).Delete\n \t \t\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store.go:429\n \t \tgithub.com/stackrox/rox/central/resourcecollection/datastore/store/postgres.(*CollectionsStoreSuite).TestStore\n \t \t\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store_test.go:95\n \t \treflect.Value.call\n \t \t\t/usr/local/go/src/reflect/value.go:556\n \t \treflect.Value.Call\n \t \t\t/usr/local/go/src/reflect/value.go:339\n \t \tgithub.com/stretchr/testify/suite.Run.func1\n \t \t\t/go/pkg/mod/github.com/stretchr/testify@v1.8.0/suite/suite.go:175\n \t \ttesting.tRunner\n \t \t\t/usr/local/go/src/testing/testing.go:1439\n \t \truntime.goexit\n \t \t\t/usr/local/go/src/runtime/asm_amd64.s:1571\n \tTest: \tTestCollectionsStore/TestStore\n store_test.go:98:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store_test.go:98\n \tError: \tShould be false\n \tTest: \tTestCollectionsStore/TestStore\n store_test.go:99:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store_test.go:99\n \tError: \tExpected nil, but got: &storage.ResourceCollection{Id:\"a\", Name:\"a\", Description:\"a\", CreatedAt:&types.Timestamp{Seconds: 1,\n \t \tNanos: 1,\n \t \t}, LastUpdated:&types.Timestamp{Seconds: 1,\n \t \tNanos: 1,\n \t \t}, CreatedBy:(*storage.SlimUser)(0xc00085fb00), UpdatedBy:(*storage.SlimUser)(0xc00085fb40), ResourceSelectors:[]*storage.ResourceSelector{(*storage.ResourceSelector)(0xc00085fb80)}, EmbeddedCollections:[]*storage.ResourceCollection_EmbeddedResourceCollection{(*storage.ResourceCollection_EmbeddedResourceCollection)(0xc0011e00f0)}, XXX_NoUnkeyedLiteral:struct {}{}, XXX_unrecognized:[]uint8(nil), XXX_sizecache:0}\n \tTest: \tTestCollectionsStore/TestStore\n store_test.go:114:\n \tError Trace:\t/go/src/github.com/stackrox/stackrox/central/resourcecollection/datastore/store/postgres/store_test.go:114\n \tError: \tNot equal:\n \t \texpected: 200\n \t \tactual : 201\n \tTest: \tTestCollectionsStore/TestStore", BuildId: "1", }, }, @@ -175,6 +166,13 @@ github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres / Te Stderr: "", Suite: "DefaultPoliciesTest", BuildId: "1", + Error: "Condition not satisfied:\n" + + "\n" + + "waitForViolation(deploymentName, policyName, 60)\n" + + "| | |\n" + + "false qadefpolstruts Apache Struts: CVE-2017-5638\n" + + "\n" + + "\tat DefaultPoliciesTest.Verify policy #policyName is triggered(DefaultPoliciesTest.groovy:181)\n", }}, tests, ) diff --git a/testdata/report.xml b/testdata/report.xml index 18ad06b..2a745ec 100644 --- a/testdata/report.xml +++ b/testdata/report.xml @@ -1,5 +1,5 @@ - + @@ -10735,138 +10735,150 @@ ok github.com/stackrox/rox/pkg/bolthelper/tests 0.083s coverage: [no statement - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -17595,7 +17607,8 @@ the login page, and log in with username "admin" and the password found in the test-token ]]> - + From 4839d6dadba8514fb09e04921780f3fab2a8d7aa Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Mon, 2 Jan 2023 07:23:19 +0100 Subject: [PATCH 3/6] Add Error field to template. --- main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main.go b/main.go index bb6799b..f5a59c0 100644 --- a/main.go +++ b/main.go @@ -263,6 +263,11 @@ const ( {{ .Stdout }} {code} {{- end }} +{{-if .Error }} +{code:title=ERROR|borderStyle=solid} +{{ .Error }} +{code} +{{- end }} || ENV || Value || | BUILD ID | [{{- .BuildId -}}|https://prow.ci.openshift.org/view/gs/origin-ci-test/logs/{{- .JobName -}}/{{- .BuildId -}}]| From f9c9f37e2549e0f6c2337e6d4e72c357e00788e8 Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Mon, 2 Jan 2023 09:17:45 +0100 Subject: [PATCH 4/6] Fix unit test failure. --- main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index f5a59c0..c054e97 100644 --- a/main.go +++ b/main.go @@ -297,12 +297,11 @@ type testCase struct { func NewTestCase(tc junit.Test, env map[string]string) testCase { jobSpec := env["JOB_SPEC"] baseLink := gjson.Get(jobSpec, "refs.base_link").String() - return testCase{ + c := testCase{ Name: tc.Name, Message: tc.Message, Stdout: tc.SystemOut, Stderr: tc.SystemErr, - Error: tc.Error.Error(), Suite: tc.Classname, BuildId: env["BUILD_ID"], Cluster: env["CLUSTER_NAME"], @@ -311,6 +310,11 @@ func NewTestCase(tc junit.Test, env map[string]string) testCase { BuildTag: env["STACKROX_BUILD_TAG"], BaseLink: baseLink, } + + if tc.Error != nil { + c.Error = tc.Error.Error() + } + return c } func (tc *testCase) description() (string, error) { From 8dfd5bb968d831862adc44579e89722d6a26d71a Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Mon, 2 Jan 2023 09:23:16 +0100 Subject: [PATCH 5/6] xxFix unit tests after rebase. --- main.go | 2 +- main_test.go | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index c054e97..92708f6 100644 --- a/main.go +++ b/main.go @@ -263,7 +263,7 @@ const ( {{ .Stdout }} {code} {{- end }} -{{-if .Error }} +{{- if .Error }} {code:title=ERROR|borderStyle=solid} {{ .Error }} {code} diff --git a/main_test.go b/main_test.go index c341127..6f9092a 100644 --- a/main_test.go +++ b/main_test.go @@ -36,17 +36,16 @@ func TestParseJunitReport(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []testCase{ { - Message: `github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests FAILED -github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh FAILED -github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets FAILED + Message: `github.com/stackrox/rox/pkg/booleanpolicy/evaluator / TestDifferentBaseTypes FAILED +github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests FAILED `, JobName: "job-name", - Suite: "github.com/stackrox/rox/sensor/kubernetes/localscanner", + Suite: "", }, }, tests) }) t.Run("dir multiple suites with threshold", func(t *testing.T) { - tests, err := findFailedTests("testdata", map[string]string{"JOB_NAME": "job-name", "BUILD_ID": "1"}, 5) + tests, err := findFailedTests("testdata", map[string]string{"JOB_NAME": "job-name", "BUILD_ID": "1"}, 3) assert.NoError(t, err) assert.ElementsMatch( @@ -54,11 +53,9 @@ github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssu []testCase{ { Message: `DefaultPoliciesTest / Verify policy Apache Struts CVE-2017-5638 is triggered FAILED +github.com/stackrox/rox/pkg/booleanpolicy/evaluator / TestDifferentBaseTypes FAILED github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests FAILED -github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh FAILED -github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests/TestSuccessfulRefresh/no_secrets FAILED github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres / TestCollectionsStore FAILED -github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres / TestCollectionsStore/TestStore FAILED `, JobName: "job-name", BuildId: "1", From ff1bfb93a3f04fdcd730cb28340d8da9e2c1515d Mon Sep 17 00:00:00 2001 From: Daniel Haus Date: Tue, 3 Jan 2023 12:33:59 +0100 Subject: [PATCH 6/6] Only group go tests. --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 92708f6..3de3a1f 100644 --- a/main.go +++ b/main.go @@ -236,7 +236,7 @@ func addSubTestToFailedTest(subTest junit.Test, failedTests []testCase, env map[ name := strings.Split(subTest.Name, "/")[0] for i, failedTest := range failedTests { // Only consider a failed test a "parent" of the test if the name matches _and_ the class name is the same. - if failedTest.Name == name && failedTest.Suite == subTest.Classname { + if isGoTest(subTest.Classname) && failedTest.Name == name && failedTest.Suite == subTest.Classname { failedTest.addSubTest(subTest) failedTests[i] = failedTest return failedTests @@ -246,6 +246,11 @@ func addSubTestToFailedTest(subTest junit.Test, failedTests []testCase, env map[ return append(failedTests, NewTestCase(subTest, env)) } +// isGoTest will verify that the corresponding classname refers to a go package by expecting the go module name as prefix. +func isGoTest(className string) bool { + return strings.HasPrefix(className, "github.com/stackrox/rox") +} + const ( desc = ` {{- if .Message }}