Skip to content

TRT-2350: Include additional OTE metadata in JUnit results #30431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+313 −11
Open

TRT-2350: Include additional OTE metadata in JUnit results #30431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 49 additions & 10 deletions pkg/test/ginkgo/junit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,40 @@ import (
"time"

"github.com/openshift/origin/pkg/test"
"github.com/openshift/origin/pkg/test/extensions"
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"

"github.com/openshift/origin/pkg/version"
)

// populateOTEMetadata adds OTE metadata attributes to a JUnit test case if available
func populateOTEMetadata(testCase *junitapi.JUnitTestCase, extensionResult *extensions.ExtensionTestResult) {
if extensionResult == nil {
return
}

// Test source information
testCase.SourceImage = extensionResult.Source.SourceImage
testCase.SourceBinary = extensionResult.Source.SourceBinary
if extensionResult.Source.Source != nil {
testCase.SourceURL = extensionResult.Source.SourceURL
testCase.SourceCommit = extensionResult.Source.Commit
}

// Set lifecycle attribute
testCase.Lifecycle = string(extensionResult.Lifecycle)

// Set start time attribute if available
if extensionResult.StartTime != nil {
testCase.StartTime = time.Time(*extensionResult.StartTime).UTC().Format(time.RFC3339)
}

// Set end time attribute if available
if extensionResult.EndTime != nil {
testCase.EndTime = time.Time(*extensionResult.EndTime).UTC().Format(time.RFC3339)
}
}

func generateJUnitTestSuiteResults(
name string,
duration time.Duration,
Expand All @@ -36,49 +65,59 @@ func generateJUnitTestSuiteResults(
case test.skipped:
s.NumTests++
s.NumSkipped++
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
testCase := &junitapi.JUnitTestCase{
Name: test.name,
SystemOut: string(test.testOutputBytes),
Duration: test.duration.Seconds(),
SkipMessage: &junitapi.SkipMessage{
Message: lastLinesUntil(string(test.testOutputBytes), 100, "skip ["),
},
})
}
populateOTEMetadata(testCase, test.extensionTestResult)
s.TestCases = append(s.TestCases, testCase)
case test.failed:
s.NumTests++
s.NumFailed++
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
testCase := &junitapi.JUnitTestCase{
Name: test.name,
SystemOut: string(test.testOutputBytes),
Duration: test.duration.Seconds(),
FailureOutput: &junitapi.FailureOutput{
Output: lastLinesUntil(string(test.testOutputBytes), 100, "fail ["),
},
})
}
populateOTEMetadata(testCase, test.extensionTestResult)
s.TestCases = append(s.TestCases, testCase)
case test.flake:
s.NumTests++
s.NumFailed++
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
failedTestCase := &junitapi.JUnitTestCase{
Name: test.name,
SystemOut: string(test.testOutputBytes),
Duration: test.duration.Seconds(),
FailureOutput: &junitapi.FailureOutput{
Output: lastLinesUntil(string(test.testOutputBytes), 100, "flake:"),
},
})
}
populateOTEMetadata(failedTestCase, test.extensionTestResult)
s.TestCases = append(s.TestCases, failedTestCase)

// also add the successful junit result:
s.NumTests++
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
successTestCase := &junitapi.JUnitTestCase{
Name: test.name,
Duration: test.duration.Seconds(),
})
}
populateOTEMetadata(successTestCase, test.extensionTestResult)
s.TestCases = append(s.TestCases, successTestCase)
case test.success:
s.NumTests++
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
testCase := &junitapi.JUnitTestCase{
Name: test.name,
Duration: test.duration.Seconds(),
})
}
populateOTEMetadata(testCase, test.extensionTestResult)
s.TestCases = append(s.TestCases, testCase)
}
}
for _, result := range syntheticTestResults {
Expand Down
256 changes: 255 additions & 1 deletion pkg/test/ginkgo/junit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
package ginkgo

import "testing"
import (
"encoding/xml"
"strings"
"testing"
"time"

"github.com/openshift-eng/openshift-tests-extension/pkg/dbtime"
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
"github.com/openshift/origin/pkg/test/extensions"
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
)

func Test_lastLines(t *testing.T) {
tests := []struct {
Expand Down Expand Up @@ -33,3 +44,246 @@ func Test_lastLines(t *testing.T) {
})
}
}

func Test_populateOTEMetadata(t *testing.T) {
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)

tests := []struct {
name string
extensionResult *extensions.ExtensionTestResult
expectedLifecycle string
expectedStartTime string
expectedEndTime string
expectedSourceImage string
expectedSourceBinary string
expectedSourceURL string
expectedSourceCommit string
}{
{
name: "nil extension result",
extensionResult: nil,
expectedLifecycle: "",
expectedStartTime: "",
expectedEndTime: "",
expectedSourceImage: "",
expectedSourceBinary: "",
expectedSourceURL: "",
expectedSourceCommit: "",
},
{
name: "complete extension result",
extensionResult: &extensions.ExtensionTestResult{
ExtensionTestResult: &extensiontests.ExtensionTestResult{
Name: "test-case",
Lifecycle: extensiontests.LifecycleBlocking,
StartTime: dbtime.Ptr(startTime),
EndTime: dbtime.Ptr(endTime),
},
Source: extensions.Source{
Source: &extension.Source{
Commit: "abc123def456",
SourceURL: "https://github.com/example/repo",
},
SourceImage: "tests",
SourceBinary: "openshift-tests",
},
},
expectedLifecycle: "blocking",
expectedStartTime: "2023-12-25T10:00:00Z",
expectedEndTime: "2023-12-25T10:05:00Z",
expectedSourceImage: "tests",
expectedSourceBinary: "openshift-tests",
expectedSourceURL: "https://github.com/example/repo",
expectedSourceCommit: "abc123def456",
},
{
name: "informing lifecycle",
extensionResult: &extensions.ExtensionTestResult{
ExtensionTestResult: &extensiontests.ExtensionTestResult{
Name: "test-case",
Lifecycle: extensiontests.LifecycleInforming,
StartTime: dbtime.Ptr(startTime),
EndTime: dbtime.Ptr(endTime),
},
Source: extensions.Source{
Source: &extension.Source{
Commit: "xyz789",
SourceURL: "https://github.com/openshift/origin",
},
SourceImage: "tests",
SourceBinary: "openshift-tests",
},
},
expectedLifecycle: "informing",
expectedStartTime: "2023-12-25T10:00:00Z",
expectedEndTime: "2023-12-25T10:05:00Z",
expectedSourceImage: "tests",
expectedSourceBinary: "openshift-tests",
expectedSourceURL: "https://github.com/openshift/origin",
expectedSourceCommit: "xyz789",
},
{
name: "missing time fields",
extensionResult: &extensions.ExtensionTestResult{
ExtensionTestResult: &extensiontests.ExtensionTestResult{
Name: "test-case",
Lifecycle: extensiontests.LifecycleBlocking,
StartTime: nil,
EndTime: nil,
},
},
expectedLifecycle: "blocking",
expectedStartTime: "",
expectedEndTime: "",
expectedSourceImage: "",
expectedSourceBinary: "",
expectedSourceURL: "",
expectedSourceCommit: "",
},
{
name: "partial source information",
extensionResult: &extensions.ExtensionTestResult{
ExtensionTestResult: &extensiontests.ExtensionTestResult{
Name: "test-case",
Lifecycle: extensiontests.LifecycleBlocking,
StartTime: dbtime.Ptr(startTime),
EndTime: dbtime.Ptr(endTime),
},
Source: extensions.Source{
Source: &extension.Source{
Commit: "abc123",
},
SourceImage: "tests",
},
},
expectedLifecycle: "blocking",
expectedStartTime: "2023-12-25T10:00:00Z",
expectedEndTime: "2023-12-25T10:05:00Z",
expectedSourceImage: "tests",
expectedSourceBinary: "",
expectedSourceURL: "",
expectedSourceCommit: "abc123",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testCase := &junitapi.JUnitTestCase{
Name: "test-case",
Duration: 300.0, // 5 minutes
}

populateOTEMetadata(testCase, tt.extensionResult)

if testCase.Lifecycle != tt.expectedLifecycle {
t.Errorf("Lifecycle = %q, want %q", testCase.Lifecycle, tt.expectedLifecycle)
}
if testCase.StartTime != tt.expectedStartTime {
t.Errorf("StartTime = %q, want %q", testCase.StartTime, tt.expectedStartTime)
}
if testCase.EndTime != tt.expectedEndTime {
t.Errorf("EndTime = %q, want %q", testCase.EndTime, tt.expectedEndTime)
}
if testCase.SourceImage != tt.expectedSourceImage {
t.Errorf("SourceImage = %q, want %q", testCase.SourceImage, tt.expectedSourceImage)
}
if testCase.SourceBinary != tt.expectedSourceBinary {
t.Errorf("SourceBinary = %q, want %q", testCase.SourceBinary, tt.expectedSourceBinary)
}
if testCase.SourceURL != tt.expectedSourceURL {
t.Errorf("SourceURL = %q, want %q", testCase.SourceURL, tt.expectedSourceURL)
}
if testCase.SourceCommit != tt.expectedSourceCommit {
t.Errorf("SourceCommit = %q, want %q", testCase.SourceCommit, tt.expectedSourceCommit)
}
})
}
}

func Test_junitXMLWithOTEAttributes(t *testing.T) {
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)

// Create a JUnit test case and populate it with OTE metadata
extensionResult := &extensions.ExtensionTestResult{
ExtensionTestResult: &extensiontests.ExtensionTestResult{
Name: "example-test",
Lifecycle: extensiontests.LifecycleBlocking,
StartTime: dbtime.Ptr(startTime),
EndTime: dbtime.Ptr(endTime),
},
Source: extensions.Source{
Source: &extension.Source{
Commit: "abc123def456789",
SourceURL: "https://github.com/openshift/origin",
},
SourceImage: "tests",
SourceBinary: "openshift-tests",
},
}

junitTestCase := &junitapi.JUnitTestCase{
Name: "example-test",
Duration: 300.0, // 5 minutes
}

// Populate the OTE metadata
populateOTEMetadata(junitTestCase, extensionResult)

// Create a test suite containing our test case
suite := &junitapi.JUnitTestSuite{
Name: "test-suite",
NumTests: 1,
Duration: 300.0,
TestCases: []*junitapi.JUnitTestCase{junitTestCase},
}

// Verify OTE metadata is present
if junitTestCase.Lifecycle != "blocking" {
t.Errorf("Lifecycle = %q, want %q", junitTestCase.Lifecycle, "blocking")
}
if junitTestCase.StartTime != "2023-12-25T10:00:00Z" {
t.Errorf("StartTime = %q, want %q", junitTestCase.StartTime, "2023-12-25T10:00:00Z")
}
if junitTestCase.EndTime != "2023-12-25T10:05:00Z" {
t.Errorf("EndTime = %q, want %q", junitTestCase.EndTime, "2023-12-25T10:05:00Z")
}
if junitTestCase.SourceImage != "tests" {
t.Errorf("SourceImage = %q, want %q", junitTestCase.SourceImage, "tests")
}
if junitTestCase.SourceBinary != "openshift-tests" {
t.Errorf("SourceBinary = %q, want %q", junitTestCase.SourceBinary, "openshift-tests")
}
if junitTestCase.SourceURL != "https://github.com/openshift/origin" {
t.Errorf("SourceURL = %q, want %q", junitTestCase.SourceURL, "https://github.com/openshift/origin")
}
if junitTestCase.SourceCommit != "abc123def456789" {
t.Errorf("SourceCommit = %q, want %q", junitTestCase.SourceCommit, "abc123def456789")
}

// Verify XML marshaling includes the new attributes
xmlData, err := xml.MarshalIndent(suite, "", " ")
if err != nil {
t.Fatalf("Failed to marshal XML: %v", err)
}

xmlString := string(xmlData)

// Check that our custom attributes are in the XML
expectedAttributes := []string{
`lifecycle="blocking"`,
`start-time="2023-12-25T10:00:00Z"`,
`end-time="2023-12-25T10:05:00Z"`,
`source-image="tests"`,
`source-binary="openshift-tests"`,
`source-url="https://github.com/openshift/origin"`,
`source-commit="abc123def456789"`,
}

for _, attr := range expectedAttributes {
if !strings.Contains(xmlString, attr) {
t.Errorf("XML does not contain expected attribute: %s\nXML output:\n%s", attr, xmlString)
}
}
}
9 changes: 9 additions & 0 deletions pkg/test/ginkgo/junitapi/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ type JUnitTestCase struct {
// Duration is the time taken in seconds to run the test
Duration float64 `xml:"time,attr"`

// OTE metadata attributes
StartTime string `xml:"start-time,attr,omitempty"`
EndTime string `xml:"end-time,attr,omitempty"`
Lifecycle string `xml:"lifecycle,attr,omitempty"`
SourceImage string `xml:"source-image,attr,omitempty"`
SourceBinary string `xml:"source-binary,attr,omitempty"`
SourceURL string `xml:"source-url,attr,omitempty"`
SourceCommit string `xml:"source-commit,attr,omitempty"`

// SkipMessage holds the reason why the test was skipped
SkipMessage *SkipMessage `xml:"skipped"`

Expand Down