orch/run: Add unit test xml scanner

requirements.txt

@@ -124,6 +124,8 @@ keystoneauth1==4.3.1
     #   python-novaclient
 lupa==1.14.1
     # via teuthology (pyproject.toml)
+lxml==4.9.3
+    # via teuthology (pyproject.toml)
 markupsafe==2.0.1
     # via jinja2
 mock==4.0.3

setup.cfg

@@ -41,6 +41,7 @@ install_requires =
     httplib2
     humanfriendly
     lupa
+    lxml
     ndg-httpsclient
     netaddr
     paramiko

teuthology/exceptions.py

@@ -212,3 +212,27 @@ class NoRemoteError(Exception):
 
     def __str__(self):
         return self.message
+
+
+class UnitTestError(Exception):
+    """
+    Exception thrown on unit test failure
+    """
+    def __init__(self, exitstatus=None, node=None, label=None, message=None):
+        self.exitstatus = exitstatus
+        self.node = node
+        self.label = label
+        self.message = message
+
+    def __str__(self):
+        prefix = "Unit test failed"
+        if self.label:
+            prefix += " ({label})".format(label=self.label)
+        if self.node:
+            prefix += " on {node}".format(node=self.node)
+        if self.exitstatus:
+            prefix += " with status {status}".format(status=self.exitstatus)
+        return "{prefix}: '{message}'".format(
+            prefix=prefix,
+            message=self.message,
+        )

teuthology/orchestra/remote.py

@@ -11,7 +11,8 @@
 from teuthology.orchestra.opsys import OS
 import teuthology.provision
 from teuthology import misc
-from teuthology.exceptions import CommandFailedError
+from teuthology.exceptions import CommandFailedError, UnitTestError
+from teuthology.util.scanner import UnitTestScanner
 from teuthology.misc import host_shortname
 import errno
 import re
@@ -523,6 +524,20 @@ def run(self, **kwargs):
         r.remote = self
         return r
 
+    def run_unit_test(self, xml_path_regex, output_yaml, **kwargs):
+        try:
+            r = self.run(**kwargs)
+        except CommandFailedError as exc:
+            if xml_path_regex:
+                error_msg = UnitTestScanner(remote=self).scan_and_write(xml_path_regex, output_yaml)
+                if error_msg:
+                    raise UnitTestError(
+                        exitstatus=exc.exitstatus, node=exc.node, 
+                        label=exc.label, message=error_msg
+                    )
+            raise exc
+        return r
+
     def _sftp_put_file(self, local_path, remote_path):
         """
         Use the paramiko.SFTPClient to put a file. Returns the remote filename.
@@ -543,12 +558,14 @@ def _sftp_get_file(self, remote_path, local_path):
         sftp.get(remote_path, local_path)
         return local_path
 
-    def _sftp_open_file(self, remote_path):
+    def _sftp_open_file(self, remote_path, mode=None):
         """
         Use the paramiko.SFTPClient to open a file. Returns a
         paramiko.SFTPFile object.
         """
         sftp = self.ssh.open_sftp()
+        if mode:
+            return sftp.open(remote_path, mode)
         return sftp.open(remote_path)
 
     def _sftp_get_size(self, remote_path):

teuthology/orchestra/test/test_remote.py

@@ -1,10 +1,12 @@
 from mock import patch, Mock, MagicMock
+from pytest import raises
 
 from io import BytesIO
 
 from teuthology.orchestra import remote
 from teuthology.orchestra import opsys
 from teuthology.orchestra.run import RemoteProcess
+from teuthology.exceptions import CommandFailedError, UnitTestError
 
 
 class TestRemote(object):
@@ -66,6 +68,23 @@ def test_run(self):
         assert result is proc
         assert result.remote is rem
 
+    @patch('teuthology.util.scanner.UnitTestScanner.scan_and_write')
+    def test_run_unit_test(self, m_scan_and_write):
+        m_transport = MagicMock()
+        m_transport.getpeername.return_value = ('name', 22)
+        self.m_ssh.get_transport.return_value = m_transport
+        m_run = MagicMock(name="run", side_effect=CommandFailedError('mocked error', 1, 'smithi'))
+        args = [
+            'something',
+            'more',
+        ]
+        rem = remote.Remote(name='[email protected]', ssh=self.m_ssh)
+        rem._runner = m_run
+        m_scan_and_write.return_value = "Error Message"
+        with raises(UnitTestError) as exc:
+            rem.run_unit_test(args=args, xml_path_regex="xml_path", output_yaml="yaml_path")
+        assert str(exc.value) == "Unit test failed on smithi with status 1: 'Error Message'"
+
     def test_hostname(self):
         m_transport = MagicMock()
         m_transport.getpeername.return_value = ('name', 22)

teuthology/util/scanner.py

@@ -0,0 +1,159 @@
+import logging
+import yaml
+from typing import Optional, Tuple
+from collections import defaultdict
+from lxml import etree
+
+log = logging.getLogger(__name__)
+
+
+class Scanner():
+    def __init__(self, remote=None) -> None:
+        self.summary_data = []
+        self.remote = remote
+
+    def _parse(self, file_content) -> Tuple[str, dict]:
+        """
+        This parses file_content and returns:
+        :returns: a message string 
+        :returns: data dictionary with additional info
+
+        Just an abstract method in Scanner class, 
+        to be defined in inherited classes. 
+        """
+        raise NotImplementedError
+
+    def scan_file(self, path: str) -> Optional[str]:
+        if not path:
+            return None
+        try:
+            file = self.remote._sftp_open_file(path, 'r')
+            file_content = file.read()
+            txt, data = self._parse(file_content)
+            if data:
+                data["file_path"] = path
+                self.summary_data += [data]
+            file.close()
+            return txt
+        except Exception as exc:
+            log.error(str(exc))
+
+    def scan_all_files(self, path_regex: str) -> [str]:
+        """
+        Scans all files matching path_regex
+        and collect additional data in self.summary_data 
+
+        :param path_regex: Regex string to find all the files which have to be scanned. 
+                           Example: /path/to/dir/*.xml
+        """
+        (_, stdout, _) = self.remote.ssh.exec_command(f'ls -d {path_regex}', timeout=200)
+
+        files = stdout.read().decode().split('\n')
+
+        extracted_txts = []
+        for fpath in files:
+            txt = self.scan_file(fpath)
+            if txt:
+                extracted_txts += [txt]
+        return extracted_txts
+
+    def write_summary(self, yaml_path: str) -> None:
+        """
+        Create yaml file locally 
+        with self.summary_data.
+        """
+        if self.summary_data and yaml_path:
+            with open(yaml_path, 'a') as f:
+                yaml.safe_dump(self.summary_data, f, default_flow_style=False)
+        else:
+            log.info("summary_data or yaml_file is empty!")
+
+
+class UnitTestScanner(Scanner):
+    def __init__(self, remote=None) -> None:
+        super().__init__(remote)
+
+    def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+        xml_tree = etree.fromstring(file_content)
+
+        failed_testcases = xml_tree.xpath('.//failure/.. | .//error/..')
+        if len(failed_testcases) == 0:
+            return None, None
+
+        exception_txt = ""
+        error_data = defaultdict(list)
+        for testcase in failed_testcases:
+            testcase_name = testcase.get("name", "test-name")
+            testcase_suitename = testcase.get("classname", "suite-name")
+            for child in testcase:
+                if child.tag in ['failure', 'error']:
+                    fault_kind = child.tag
+                    reason = child.get('message', 'No message found in xml output, check logs.')
+                    short_reason = (reason[:200].strip() + '...') if len(reason) > 200 else reason.strip()
+                    error_data[testcase_suitename] += [{
+                            "kind": fault_kind, 
+                            "testcase": testcase_name,
+                            "message": reason,
+                        }]
+                    if not exception_txt:
+                        exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.'
+
+        return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) }
+
+    @property
+    def num_of_total_failures(self):
+        total_failed_testcases = 0
+        if self.summary_data:
+            for file_data in self.summary_data:
+                failed_tests = file_data.get("num_of_failures", 0)
+                total_failed_testcases += failed_tests
+        return total_failed_testcases
+
+    def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]:
+        """
+        Scan all files matching 'path_regex'
+        and write summary in 'summary_path'.
+        """
+        try:
+            errors = self.scan_all_files(path_regex)
+            self.write_summary(summary_path)
+            if errors:
+                count = self.num_of_total_failures
+                return f"(total {count} failed) " + errors[0]
+        except Exception as scanner_exc:
+            log.error(str(scanner_exc))
+
+
+class ValgrindScanner(Scanner):
+    def __init__(self, remote=None) -> None:
+        super().__init__(remote)
+
+    def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+        xml_tree = etree.fromstring(file_content)
+        if not xml_tree:
+            return None, None
+
+        error_tree = xml_tree.find('error')
+        if error_tree is None:
+            return None, None
+
+        error_data = {
+            "kind": error_tree.findtext("kind"),
+            "traceback": [],
+        }
+        for frame in error_tree.xpath("stack/frame"):
+            if len(error_data["traceback"]) >= 5:
+                break
+            curr_frame = {
+                "file": f"{frame.findtext('dir', '')}/{frame.findtext('file', '')}",
+                "line": frame.findtext("line", ''),
+                "function": frame.findtext("fn", ''),
+            }
+            error_data["traceback"].append(curr_frame)
+
+        traceback_functions = "\n".join(
+                frame.get("function", "N/A") 
+                for frame in error_data["traceback"][:3]
+            )
+        exception_text = f"valgrind error: {error_data['kind']}\n{traceback_functions}"
+        return exception_text, error_data

teuthology/util/test/files/test_unit_test.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="nosetests" tests="644" errors="0" failures="1" skip="79">
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_response" time="3.205"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_wildcard" time="3.081"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_header_option" time="3.119"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_set_bucket_tagging" time="0.059"><failure type="builtins.AssertionError" message="'NoSuchTagSetError' != 'NoSuchTagSet'"></failure></testcase>
+</testsuite>

teuthology/util/test/files/test_valgrind.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<valgrindoutput>
+<error>
+  <unique>0x870fc</unique>
+  <tid>1</tid>
+  <kind>Leak_DefinitelyLost</kind>
+  <xwhat>
+    <text>1,234 bytes in 1 blocks are definitely lost in loss record 198 of 201</text>
+    <leakedbytes>1234</leakedbytes>
+    <leakedblocks>1</leakedblocks>
+  </xwhat>
+  <stack>
+    <frame>
+      <ip>0x4C39B6F</ip>
+      <obj>/usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so</obj>
+      <fn>operator new[](unsigned long)</fn>
+      <dir>/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc</dir>
+      <file>vg_replace_malloc.c</file>
+      <line>640</line>
+    </frame>
+    <frame>
+      <ip>0xF3F4B5</ip>
+      <obj>/usr/bin/ceph-osd</obj>
+      <fn>ceph::common::leak_some_memory()</fn>
+      <dir>/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common</dir>
+      <file>ceph_context.cc</file>
+      <line>510</line>
+    </frame>
+  </stack>
+</error>
+</valgrindoutput>