Skip to content

Commit

Permalink
Generate XUnit Polarion flavor using JUnit report plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
seberm committed Aug 28, 2024
1 parent 45811c2 commit 0702405
Show file tree
Hide file tree
Showing 4 changed files with 332 additions and 71 deletions.
122 changes: 116 additions & 6 deletions tests/report/polarion/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,124 @@ rlJournalStart
rlRun "set -o pipefail"
rlPhaseEnd

rlPhaseStartTest
rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlPhaseStartTest 'Test the properties gets propagated to testsuites correctly'
rlRun "tmt run -avr execute report -h polarion --no-upload --template mytemplate --project-id RHELBASEOS --planned-in RHEL-9.1.0 --arch x86_64 --description mydesc --assignee myassignee --pool-team mypoolteam --platform myplatform --build mybuild --sample-image mysampleimage --logs mylogslocation --compose-id mycomposeid --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep "1 test passed, 1 test failed and 1 error" "output"
rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"
rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml"

# testsuites and testsuite tag attributes
rlAssertGrep '<testsuites disabled="0" errors="1" failures="1" tests="3"' "xunit.xml"
rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
# Main testsuite properties
rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-project-span-ids" value="RHELBASEOS,RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-title" value="plan_[0-9]\+"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-template-id" value="mytemplate"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-user-id" value="' "xunit.xml"

# Custom testsuite properties
rlAssertGrep '<property name="polarion-custom-description" value="mydesc"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-assignee" value="myassignee"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-poolteam" value="mypoolteam"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="x86_64"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-platform" value="myplatform"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-build" value="mybuild"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-sampleimage" value="mysampleimage"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-logs" value="mylogslocation"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-composeid" value="mycomposeid"/>' "xunit.xml"

# The testcase properties
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10913"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10915"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Test the facts properties'
rlRun "tmt run -avr execute report -h polarion --no-upload --use-facts --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'The "None" string should never be in a property value'
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id '' --planned-in '' --template '' --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertNotGrep 'value="None"' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Check the plugin behavior based on setting ENV variables'
rlRun "export \
TMT_PLUGIN_REPORT_POLARION_PROJECT_ID=myprojectid \
TMT_PLUGIN_REPORT_POLARION_TITLE=mytitle \
TMT_PLUGIN_REPORT_POLARION_DESCRIPTION=mydesc \
TMT_PLUGIN_REPORT_POLARION_TEMPLATE=mytemplate \
TMT_PLUGIN_REPORT_POLARION_PLANNED_IN=myplannedin \
TMT_PLUGIN_REPORT_POLARION_ASSIGNEE=myassignee \
TMT_PLUGIN_REPORT_POLARION_POOL_TEAM=mypoolteam \
TMT_PLUGIN_REPORT_POLARION_ARCH=x86_64 \
TMT_PLUGIN_REPORT_POLARION_PLATFORM=myplatform \
TMT_PLUGIN_REPORT_POLARION_BUILD=mybuild \
TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE=mysampleimage \
TMT_PLUGIN_REPORT_POLARION_LOGS=mylogslocation \
TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID=mycomposeid \
"

rlRun "tmt run -avr execute report -h polarion --no-upload --file xunit.xml 2>&1 >/dev/null | tee output" 2
# Main testsuite properties
rlAssertGrep '<property name="polarion-project-id" value="myprojectid"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-project-span-ids" value="myprojectid,RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-title" value="mytitle"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-template-id" value="mytemplate"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-user-id" value="' "xunit.xml"

# Custom testsuite properties
rlAssertGrep '<property name="polarion-custom-description" value="mydesc"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="myplannedin"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-assignee" value="myassignee"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-poolteam" value="mypoolteam"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="x86_64"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-platform" value="myplatform"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-build" value="mybuild"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-sampleimage" value="mysampleimage"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-logs" value="mylogslocation"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-composeid" value="mycomposeid"/>' "xunit.xml"

# The testcase properties
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10913"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10915"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Check the plugin behavior based on TMT_PLUGIN_REPORT_POLARION_USE_FACTS env variable'
# Make sure all ENV variables are unset
rlRun "unset \
TMT_PLUGIN_REPORT_POLARION_PROJECT_ID \
TMT_PLUGIN_REPORT_POLARION_TITLE \
TMT_PLUGIN_REPORT_POLARION_DESCRIPTION \
TMT_PLUGIN_REPORT_POLARION_TEMPLATE \
TMT_PLUGIN_REPORT_POLARION_PLANNED_IN \
TMT_PLUGIN_REPORT_POLARION_ASSIGNEE \
TMT_PLUGIN_REPORT_POLARION_POOL_TEAM \
TMT_PLUGIN_REPORT_POLARION_ARCH \
TMT_PLUGIN_REPORT_POLARION_PLATFORM \
TMT_PLUGIN_REPORT_POLARION_BUILD \
TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE \
TMT_PLUGIN_REPORT_POLARION_LOGS \
TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID \
"

# The facts must not be set
rlRun "export TMT_PLUGIN_REPORT_POLARION_USE_FACTS=''"
rlRun "tmt run -avr execute report -h polarion --no-upload --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertNotGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlAssertNotGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"

# The facts must be set
rlRun "export TMT_PLUGIN_REPORT_POLARION_USE_FACTS='yes'"
rlRun "tmt run -avr execute report -h polarion --no-upload --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"
rlPhaseEnd

rlPhaseStartCleanup
Expand Down
97 changes: 97 additions & 0 deletions tmt/steps/report/junit/schemas/polarion.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema supports only a subset of the features provided by the
`xml-junit` library. Additionally, many attributes are explicitly set as
required. This is intentional to limit the currently supported features of
the TMT Polarion report plugin .
The Polarion `xunit.xml` is almost the same as default output of junit
report plugin but it must allow definition of `properties` inside of
`testsuites` (NOT `testsuite`) and `testcase`.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="properties">
<xs:complexType>
<xs:sequence>
<xs:element ref="property" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="property">
<xs:complexType>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="value" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="1" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="optional"/>
<xs:attribute name="failures" type="xs:string" use="optional"/>
<xs:attribute name="disabled" type="xs:string" use="optional"/>
<xs:attribute name="errors" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>
</xs:schema>
10 changes: 10 additions & 0 deletions tmt/steps/report/junit/templates/polarion.xml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{% extends "_base.xml.j2" %}

{% block testsuites %}
{{ super() }}

{# Polarion XUnit must include the properties section in testsuites tag #}
{% with properties=TESTSUITES_PROPERTIES %}
{% include "includes/_properties.xml.j2" %}
{% endwith %}
{% endblock %}
Loading

0 comments on commit 0702405

Please sign in to comment.