From 018b2148d01dcc3212b5e1eddb1c3f813b5c5954 Mon Sep 17 00:00:00 2001 From: Otto Sabart Date: Wed, 7 Aug 2024 09:00:00 +0200 Subject: [PATCH] Generate XUnit Polarion flavor using JUnit report plugin --- tests/report/polarion/test.sh | 95 +++++++++- tmt/steps/report/junit/schemas/polarion.xsd | 97 ++++++++++ .../report/junit/templates/polarion.xml.j2 | 10 ++ tmt/steps/report/polarion.py | 166 +++++++++++------- 4 files changed, 299 insertions(+), 69 deletions(-) create mode 100644 tmt/steps/report/junit/schemas/polarion.xsd create mode 100644 tmt/steps/report/junit/templates/polarion.xml.j2 diff --git a/tests/report/polarion/test.sh b/tests/report/polarion/test.sh index d36eddc5a2..0c65890c53 100755 --- a/tests/report/polarion/test.sh +++ b/tests/report/polarion/test.sh @@ -7,14 +7,97 @@ rlJournalStart rlRun "set -o pipefail" rlPhaseEnd - rlPhaseStartTest - rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2 + rlPhaseStartTest 'Test the properties gets propagated to testsuites correctly' + rlRun "tmt run -avr execute report -h polarion --no-upload --template mytemplate --project-id RHELBASEOS --planned-in RHEL-9.1.0 --arch x86_64 --description mydesc --assignee myassignee --pool-team mypoolteam --platform myplatform --build mybuild --sample-image mysampleimage --logs mylogslocation --compose-id mycomposeid --file xunit.xml 2>&1 >/dev/null | tee output" 2 rlAssertGrep "1 test passed, 1 test failed and 1 error" "output" - rlAssertGrep '' "xunit.xml" - rlAssertGrep '' "xunit.xml" - rlAssertGrep '' "xunit.xml" rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml" + + # testsuites and testsuite tag attributes + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + + # The testcase properties + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlPhaseEnd + + rlPhaseStartTest 'Test the facts properties' + rlRun "tmt run -avr execute report -h polarion --no-upload --use-facts --file xunit.xml 2>&1 >/dev/null | tee output" 2 + rlAssertGrep '&1 >/dev/null | tee output" 2 + rlAssertNotGrep 'value="None"' "xunit.xml" + rlPhaseEnd + + rlPhaseStartTest 'Check the plugin behavior based on setting ENV variables' + rlRun "export \ + TMT_PLUGIN_REPORT_POLARION_PROJECT_ID=myprojectid \ + TMT_PLUGIN_REPORT_POLARION_TITLE=mytitle \ + TMT_PLUGIN_REPORT_POLARION_DESCRIPTION=mydesc \ + TMT_PLUGIN_REPORT_POLARION_TEMPLATE=mytemplate \ + TMT_PLUGIN_REPORT_POLARION_PLANNED_IN=myplannedin \ + TMT_PLUGIN_REPORT_POLARION_ASSIGNEE=myassignee \ + TMT_PLUGIN_REPORT_POLARION_POOL_TEAM=mypoolteam \ + TMT_PLUGIN_REPORT_POLARION_ARCH=x86_64 \ + TMT_PLUGIN_REPORT_POLARION_PLATFORM=myplatform \ + TMT_PLUGIN_REPORT_POLARION_BUILD=mybuild \ + TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE=mysampleimage \ + TMT_PLUGIN_REPORT_POLARION_LOGS=mylogslocation \ + TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID=mycomposeid \ + " + + # TODO: how to check these? + # TMT_PLUGIN_REPORT_POLARION_UPLOAD=False + # TMT_PLUGIN_REPORT_POLARION_USE_FACTS=False + + rlRun "tmt run -avr execute report -h polarion --no-upload --file xunit.xml 2>&1 >/dev/null | tee output" 2 + # Main testsuite properties + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + + # The testcase properties + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" + rlAssertGrep '' "xunit.xml" rlPhaseEnd rlPhaseStartCleanup diff --git a/tmt/steps/report/junit/schemas/polarion.xsd b/tmt/steps/report/junit/schemas/polarion.xsd new file mode 100644 index 0000000000..603d77b55f --- /dev/null +++ b/tmt/steps/report/junit/schemas/polarion.xsd @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tmt/steps/report/junit/templates/polarion.xml.j2 b/tmt/steps/report/junit/templates/polarion.xml.j2 new file mode 100644 index 0000000000..a1e86d1de2 --- /dev/null +++ b/tmt/steps/report/junit/templates/polarion.xml.j2 @@ -0,0 +1,10 @@ +{% extends "_base.xml.j2" %} + +{% block testsuites %} + {{ super() }} + + {# Polarion XUnit must include the properties section in testsuites tag #} + {% with properties=TESTSUITES_PROPERTIES %} + {% include "includes/_properties.xml.j2" %} + {% endwith %} +{% endblock %} diff --git a/tmt/steps/report/polarion.py b/tmt/steps/report/polarion.py index c6be799faf..9593675cd8 100644 --- a/tmt/steps/report/polarion.py +++ b/tmt/steps/report/polarion.py @@ -2,16 +2,16 @@ import datetime import os from typing import Optional -from xml.etree import ElementTree from requests import post import tmt import tmt.steps import tmt.steps.report +import tmt.utils from tmt.utils import Path, field -from .junit import make_junit_xml +from .junit import ResultsContext, make_junit_xml DEFAULT_NAME = 'xunit.xml' @@ -23,7 +23,7 @@ class ReportPolarionData(tmt.steps.report.ReportStepData): option='--file', metavar='FILE', help='Path to the file to store xUnit in.', - normalize=lambda key_address, raw_value, logger: Path(raw_value) if raw_value else None) + normalize=tmt.utils.normalize_path) upload: bool = field( default=True, @@ -31,6 +31,7 @@ class ReportPolarionData(tmt.steps.report.ReportStepData): is_flag=True, show_default=True, help="Whether to upload results to Polarion." + # TODO: Undocumented: TMT_PLUGIN_REPORT_POLARION_UPLOAD ) project_id: Optional[str] = field( @@ -39,7 +40,9 @@ class ReportPolarionData(tmt.steps.report.ReportStepData): metavar='ID', help=""" Use specific Polarion project ID, - also uses environment variable TMT_PLUGIN_REPORT_POLARION_PROJECT_ID. + also uses environment variable TMT_PLUGIN_REPORT_POLARION_PROJECT_ID. If no project ID + is found, the project-id is taken from default project of Polarion user session as a + last resort. """ ) @@ -79,6 +82,7 @@ class ReportPolarionData(tmt.steps.report.ReportStepData): is_flag=True, show_default=True, help='Use hostname and arch from guest facts.' + # TODO: Undocumented TMT_PLUGIN_REPORT_POLARION_USE_FACTS ) planned_in: Optional[str] = field( @@ -180,6 +184,13 @@ class ReportPolarionData(tmt.steps.report.ReportStepData): help='FIPS mode enabled or disabled for this run.' ) + prettify: bool = field( + default=True, + option=('--prettify / --no-prettify'), + is_flag=True, + show_default=True, + help="Enable the XML pretty print for generated XUnit file.") + include_output_log: bool = field( default=True, option=('--include-output-log / --no-include-output-log'), @@ -204,7 +215,6 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: from tmt.export.polarion import find_polarion_case_ids, import_polarion import_polarion() from tmt.export.polarion import PolarionWorkItem - assert PolarionWorkItem title = self.data.title if not title: @@ -213,93 +223,120 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: self.step.plan.name.rsplit('/', 1)[1] + '_' + # Polarion server running with UTC timezone datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y%m%d%H%M%S")) + title = title.replace('-', '_') - project_id = self.data.project_id or os.getenv('TMT_PLUGIN_REPORT_POLARION_PROJECT_ID') template = self.data.template or os.getenv('TMT_PLUGIN_REPORT_POLARION_TEMPLATE') + project_id = self.data.project_id or os.getenv( + 'TMT_PLUGIN_REPORT_POLARION_PROJECT_ID', + PolarionWorkItem._session.default_project) + + # The project_id is required + if not project_id: + raise tmt.utils.ReportError( + "The Polarion project ID could not be determined. Consider setting it using " + "'--project-id' argument or by setting 'TMT_PLUGIN_REPORT_POLARION_PROJECT_ID' " + "environment variable.") + # TODO: try use self.data instead - but these fields are not optional, they do have # default values, do envvars even have any effect at all?? upload = self.get('upload', os.getenv('TMT_PLUGIN_REPORT_POLARION_UPLOAD')) use_facts = self.get('use-facts', os.getenv('TMT_PLUGIN_REPORT_POLARION_USE_FACTS')) - other_testrun_fields = [ - 'description', 'planned_in', 'assignee', 'pool_team', 'arch', 'platform', 'build', - 'sample_image', 'logs', 'compose_id', 'fips'] - xml_data = make_junit_xml( - phase=self, - - # TODO: Explicitly use 'default' flavor until the 'polarion' flavor - # gets implemented in junit report plugin. - # flavor='polarion', - flavor='default', - - include_output_log=self.data.include_output_log) + other_testrun_fields = [ + 'description', + 'planned_in', + 'assignee', + 'pool_team', + 'arch', + 'platform', + 'build', + 'sample_image', + 'logs', + 'compose_id', + 'fips'] + + testsuites_properties = {} - # S314: Any potential xml parser vulnerability mitigation would require defusedxml package - xml_tree = ElementTree.fromstring(xml_data) # noqa: S314 - properties = { - 'polarion-project-id': project_id, - 'polarion-user-id': PolarionWorkItem._session.user_id, - 'polarion-testrun-title': title, - 'polarion-project-span-ids': project_id} for tr_field in other_testrun_fields: param = self.get(tr_field, os.getenv(f'TMT_PLUGIN_REPORT_POLARION_{tr_field.upper()}')) # TODO: remove the os.getenv when envvars in click work with steps in plans as well # as with steps on cmdline if param: - properties[f"polarion-custom-{tr_field.replace('_', '')}"] = param + testsuites_properties[f"polarion-custom-{tr_field.replace('_', '')}"] = param + if use_facts: - properties['polarion-custom-hostname'] = \ - self.step.plan.provision.guests()[0].primary_address - properties['polarion-custom-arch'] = self.step.plan.provision.guests()[0].facts.arch + guests = self.step.plan.provision.guests() + try: + testsuites_properties['polarion-custom-hostname'] = guests[0].primary_address + testsuites_properties['polarion-custom-arch'] = guests[0].facts.arch + except IndexError as e: + raise tmt.utils.ReportError( + f'Failed to get the Polarion facts from `step.plan.provision.guests()`: {e}') + if template: - properties['polarion-testrun-template-id'] = template + testsuites_properties['polarion-testrun-template-id'] = template + logs = os.getenv('TMT_REPORT_ARTIFACTS_URL') - if logs and 'polarion-custom-logs' not in properties: - properties['polarion-custom-logs'] = logs - testsuites_properties = ElementTree.SubElement(xml_tree, 'properties') - for name, value in properties.items(): - ElementTree.SubElement(testsuites_properties, 'property', attrib={ - 'name': name, 'value': str(value)}) - - testsuite = xml_tree.find('testsuite') - project_span_ids = xml_tree.find( - '*property[@name="polarion-project-span-ids"]') - - for result in self.step.plan.execute.results(): + if logs and 'polarion-custom-logs' not in testsuites_properties: + testsuites_properties['polarion-custom-logs'] = logs + + project_span_ids: list[str] = [] + + results_context = ResultsContext(self.step.plan.execute.results()) + + for result in results_context: if not result.ids or not any(result.ids.values()): self.warn( f"Test Case '{result.name}' is not exported to Polarion, " "please run 'tmt tests export --how polarion' on it.") continue + work_item_id, test_project_id = find_polarion_case_ids(result.ids) if test_project_id is None: self.warn(f"Test case '{result.name}' missing or not found in Polarion.") continue - assert work_item_id is not None - assert project_span_ids is not None + if work_item_id is None: + raise tmt.utils.ReportError( + f"Work item ID missing or not found in Polarion (Test case: '{result.name}', " + "Test project ID: '{test_project_id}).") - if test_project_id not in project_span_ids.attrib['value']: - project_span_ids.attrib['value'] += f',{test_project_id}' + if test_project_id not in project_span_ids: + project_span_ids.append(test_project_id) - test_properties = { + testcase_properties = { 'polarion-testcase-id': work_item_id, - 'polarion-testcase-project-id': test_project_id} + 'polarion-testcase-project-id': test_project_id, + } - assert testsuite is not None - test_case = testsuite.find(f"*[@name='{result.name}']") - assert test_case is not None - properties_elem = ElementTree.SubElement(test_case, 'properties') - for name, value in test_properties.items(): - ElementTree.SubElement(properties_elem, 'property', attrib={ - 'name': name, 'value': value}) + result.set_properties(testcase_properties) assert self.workdir is not None + testsuites_properties.update({ + 'polarion-project-id': project_id, + 'polarion-user-id': PolarionWorkItem._session.user_id, + 'polarion-testrun-title': title, + 'polarion-project-span-ids': ','.join([project_id, *project_span_ids])}) + + xml_data = make_junit_xml( + phase=self, + flavor='polarion', + prettify=self.data.prettify, + include_output_log=self.data.include_output_log, + results_context=results_context, + TESTSUITES_PROPERTIES=[{'name': name, 'value': value} + for name, value in testsuites_properties.items()], + ) + f_path = self.data.file or self.workdir / DEFAULT_NAME - with open(f_path, 'wb') as fw: - ElementTree.ElementTree(xml_tree).write(fw, xml_declaration=True) + + try: + with open(f_path, 'w') as fw: + fw.write(xml_data) + except Exception as error: + raise tmt.utils.ReportError(f"Failed to write the output '{f_path}' ({error}).") if upload: server_url = str(PolarionWorkItem._session._server.url) @@ -311,14 +348,17 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: PolarionWorkItem._session.password) response = post( - polarion_import_url, auth=auth, - files={'file': ('xunit.xml', ElementTree.tostring(xml_tree))}, timeout=10 - ) + polarion_import_url, + auth=auth, + files={ + 'file': ('xunit.xml', xml_data), + }, + timeout=10) self.info( f'Response code is {response.status_code} with text: {response.text}') else: - self.info("Polarion upload can be done manually using command:") + self.info('Polarion upload can be done manually using command:') self.info( - "curl -k -u : -X POST -F file=@ " - "/polarion/import/xunit") + 'curl -k -u : -X POST -F file=@ ' + '/polarion/import/xunit') self.info('xUnit file saved at', f_path, 'yellow')