From 5d8ce219f11706ce5cf0dfd33fb3383084ae7d89 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Sat, 1 Oct 2022 16:46:52 +0530 Subject: [PATCH 1/3] add Scanner, UnitTestScanner, ValgrindScanner 1. add 'run_unit_test' to Remote 2. create util/scanner.py 3. new exception: UnitTestError 4. add `lxml` dependency in setup.cfg Signed-off-by: Vallari Agrawal --- requirements.txt | 2 + setup.cfg | 1 + teuthology/exceptions.py | 24 +++ teuthology/orchestra/remote.py | 21 ++- .../test/xml_files/test_scan_nose.xml | 73 +++++++++ teuthology/util/scanner.py | 149 ++++++++++++++++++ 6 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 teuthology/orchestra/test/xml_files/test_scan_nose.xml create mode 100644 teuthology/util/scanner.py diff --git a/requirements.txt b/requirements.txt index f56344bd85..56af1c00e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -124,6 +124,8 @@ keystoneauth1==4.3.1 # python-novaclient lupa==1.14.1 # via teuthology (pyproject.toml) +lxml==4.9.3 + # via teuthology (pyproject.toml) markupsafe==2.0.1 # via jinja2 mock==4.0.3 diff --git a/setup.cfg b/setup.cfg index 6b9d2c2c12..9d1c0134e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ install_requires = httplib2 humanfriendly lupa + lxml ndg-httpsclient netaddr paramiko diff --git a/teuthology/exceptions.py b/teuthology/exceptions.py index cbe8b5941f..da38343545 100644 --- a/teuthology/exceptions.py +++ b/teuthology/exceptions.py @@ -212,3 +212,27 @@ class NoRemoteError(Exception): def __str__(self): return self.message + + +class UnitTestError(Exception): + """ + Exception thrown on unit test failure + """ + def __init__(self, exitstatus=None, node=None, label=None, message=None): + self.exitstatus = exitstatus + self.node = node + self.label = label + self.message = message + + def __str__(self): + prefix = "Unit test failed" + if self.label: + prefix += " ({label})".format(label=self.label) + if self.node: + prefix += " on {node}".format(node=self.node) + if self.exitstatus: + prefix += " with status {status}".format(status=self.exitstatus) + return "{prefix}: '{message}'".format( + prefix=prefix, + message=self.message, + ) diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 0392acf87e..ce77a519cf 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -11,7 +11,8 @@ from teuthology.orchestra.opsys import OS import teuthology.provision from teuthology import misc -from teuthology.exceptions import CommandFailedError +from teuthology.exceptions import CommandFailedError, UnitTestError +from teuthology.util.scanner import UnitTestScanner from teuthology.misc import host_shortname import errno import re @@ -523,6 +524,20 @@ def run(self, **kwargs): r.remote = self return r + def run_unit_test(self, xml_path_regex, output_yaml, **kwargs): + try: + r = self.run(**kwargs) + except CommandFailedError as exc: + if xml_path_regex: + error_msg = UnitTestScanner(remote=self).scan_and_write(xml_path_regex, output_yaml) + if error_msg: + raise UnitTestError( + exitstatus=exc.exitstatus, node=exc.node, + label=exc.label, message=error_msg + ) + raise exc + return r + def _sftp_put_file(self, local_path, remote_path): """ Use the paramiko.SFTPClient to put a file. Returns the remote filename. @@ -543,12 +558,14 @@ def _sftp_get_file(self, remote_path, local_path): sftp.get(remote_path, local_path) return local_path - def _sftp_open_file(self, remote_path): + def _sftp_open_file(self, remote_path, mode=None): """ Use the paramiko.SFTPClient to open a file. Returns a paramiko.SFTPFile object. """ sftp = self.ssh.open_sftp() + if mode: + return sftp.open(remote_path, mode) return sftp.open(remote_path) def _sftp_get_size(self, remote_path): diff --git a/teuthology/orchestra/test/xml_files/test_scan_nose.xml b/teuthology/orchestra/test/xml_files/test_scan_nose.xml new file mode 100644 index 0000000000..d0fab48161 --- /dev/null +++ b/teuthology/orchestra/test/xml_files/test_scan_nose.xml @@ -0,0 +1,73 @@ + + + + + +> begin captured logging << -------------------- +botocore.hooks: DEBUG: Event choose-service-name: calling handler +PUT +/test-client.0-2txq2dyjghs0vdf-335 + +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb +botocore.auth: DEBUG: Signature: +8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 +botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 +botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'' +botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler +botocore.retryhandler: DEBUG: No retry needed. +GET +/test-client.0-2txq2dyjghs0vdf-335 +tagging= +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 +botocore.auth: DEBUG: Signature: +a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 +botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 +botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'NoSuchTagSetErrortest-client.0-2txq2dyjghs0vdf-335tx00000ebc589e4bcad8d86-0063354035-1157-default1157-default-default' +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler +botocore.retryhandler: DEBUG: No retry needed. +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler > +--------------------- >> end captured logging << ---------------------]]> + \ No newline at end of file diff --git a/teuthology/util/scanner.py b/teuthology/util/scanner.py new file mode 100644 index 0000000000..1e140d618c --- /dev/null +++ b/teuthology/util/scanner.py @@ -0,0 +1,149 @@ +import logging +import yaml +from typing import Optional, Tuple +from collections import defaultdict +from lxml import etree + +log = logging.getLogger(__name__) + + +class Scanner(): + def __init__(self, remote=None) -> None: + self.summary_data = [] + self.remote = remote + + def _parse(self, file_content) -> Tuple[str, dict]: + """ + This parses file_content and returns: + :returns: a message string + :returns: data dictionary with additional info + + Just an abstract method in Scanner class, + to be defined in inherited classes. + """ + raise NotImplementedError + + def scan_file(self, path: str) -> Optional[str]: + if not path: + return None + try: + file = self.remote._sftp_open_file(path, 'r') + file_content = file.read() + txt, data = self._parse(file_content) + if data: + data["file_path"] = path + self.summary_data += [data] + file.close() + return txt + except Exception as exc: + log.error(str(exc)) + + def scan_all_files(self, path_regex: str) -> [str]: + """ + Scans all files matching path_regex + and collect additional data in self.summary_data + + :param path_regex: Regex string to find all the files which have to be scanned. + Example: /path/to/dir/*.xml + """ + (_, stdout, _) = self.remote.ssh.exec_command(f'ls -d {path_regex}', timeout=200) + + files = stdout.read().decode().split('\n') + + extracted_txts = [] + for fpath in files: + txt = self.scan_file(fpath) + if txt: + extracted_txts += [txt] + return extracted_txts + + def write_summary(self, yaml_path: str) -> None: + """ + Create yaml file locally + with self.summary_data. + """ + if self.summary_data and yaml_path: + with open(yaml_path, 'a') as f: + yaml.safe_dump(self.summary_data, f, default_flow_style=False) + else: + log.info("summary_data or yaml_file is empty!") + + +class UnitTestScanner(Scanner): + def __init__(self, remote=None) -> None: + super().__init__(remote) + + def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: + xml_tree = etree.fromstring(file_content) + + failed_testcases = xml_tree.xpath('.//failure/.. | .//error/..') + if len(failed_testcases) == 0: + return None, None + + exception_txt = "" + error_data = defaultdict(list) + for testcase in failed_testcases: + testcase_name = testcase.get("name", "test-name") + testcase_suitename = testcase.get("classname", "suite-name") + for child in testcase: + if child.tag in ['failure', 'error']: + fault_kind = child.tag + reason = child.get('message', 'No message found in xml output, check logs.') + short_reason = reason[:200] + error_data[testcase_suitename] += [{ + "kind": fault_kind, + "testcase": testcase_name, + "message": reason, + }] + if not exception_txt: + exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.' + + return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) } + + def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]: + """ + Scan all files matching 'path_regex' + and write summary in 'summary_path'. + """ + try: + errors = self.scan_all_files(path_regex) + self.write_summary(summary_path) + if errors: + return errors[0] + except Exception as scanner_exc: + log.error(str(scanner_exc)) + + +class ValgrindScanner(Scanner): + def __init__(self, remote=None) -> None: + super().__init__(remote) + + def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: + xml_tree = etree.fromstring(file_content) + if not xml_tree: + return None, None + + error_tree = xml_tree.find('error') + if error_tree is None: + return None, None + + error_data = { + "kind": error_tree.findtext("kind"), + "traceback": [], + } + for frame in error_tree.xpath("stack/frame"): + if len(error_data["traceback"]) >= 5: + break + curr_frame = { + "file": f"{frame.findtext('dir', '')}/{frame.findtext('file', '')}", + "line": frame.findtext("line", ''), + "function": frame.findtext("fn", ''), + } + error_data["traceback"].append(curr_frame) + + traceback_functions = "\n".join( + frame.get("function", "N/A") + for frame in error_data["traceback"][:3] + ) + exception_text = f"valgrind error: {error_data['kind']}\n{traceback_functions}" + return exception_text, error_data From bfbd419090e97586a313161d0b041f1312a36051 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Tue, 19 Sep 2023 20:34:28 +0530 Subject: [PATCH 2/3] add utils/tests/test_scanner.py and test_run_unit_test in test_remote.py Signed-off-by: Vallari Agrawal --- teuthology/orchestra/test/test_remote.py | 19 ++ .../test/xml_files/test_scan_nose.xml | 73 ------- teuthology/util/test/files/test_unit_test.xml | 7 + teuthology/util/test/files/test_valgrind.xml | 31 +++ teuthology/util/test/test_scanner.py | 191 ++++++++++++++++++ 5 files changed, 248 insertions(+), 73 deletions(-) delete mode 100644 teuthology/orchestra/test/xml_files/test_scan_nose.xml create mode 100644 teuthology/util/test/files/test_unit_test.xml create mode 100644 teuthology/util/test/files/test_valgrind.xml create mode 100644 teuthology/util/test/test_scanner.py diff --git a/teuthology/orchestra/test/test_remote.py b/teuthology/orchestra/test/test_remote.py index 76eae68bad..a953835e7c 100644 --- a/teuthology/orchestra/test/test_remote.py +++ b/teuthology/orchestra/test/test_remote.py @@ -1,10 +1,12 @@ from mock import patch, Mock, MagicMock +from pytest import raises from io import BytesIO from teuthology.orchestra import remote from teuthology.orchestra import opsys from teuthology.orchestra.run import RemoteProcess +from teuthology.exceptions import CommandFailedError, UnitTestError class TestRemote(object): @@ -66,6 +68,23 @@ def test_run(self): assert result is proc assert result.remote is rem + @patch('teuthology.util.scanner.UnitTestScanner.scan_and_write') + def test_run_unit_test(self, m_scan_and_write): + m_transport = MagicMock() + m_transport.getpeername.return_value = ('name', 22) + self.m_ssh.get_transport.return_value = m_transport + m_run = MagicMock(name="run", side_effect=CommandFailedError('mocked error', 1, 'smithi')) + args = [ + 'something', + 'more', + ] + rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh) + rem._runner = m_run + m_scan_and_write.return_value = "Error Message" + with raises(UnitTestError) as exc: + rem.run_unit_test(args=args, xml_path_regex="xml_path", output_yaml="yaml_path") + assert str(exc.value) == "Unit test failed on smithi with status 1: 'Error Message'" + def test_hostname(self): m_transport = MagicMock() m_transport.getpeername.return_value = ('name', 22) diff --git a/teuthology/orchestra/test/xml_files/test_scan_nose.xml b/teuthology/orchestra/test/xml_files/test_scan_nose.xml deleted file mode 100644 index d0fab48161..0000000000 --- a/teuthology/orchestra/test/xml_files/test_scan_nose.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - -> begin captured logging << -------------------- -botocore.hooks: DEBUG: Event choose-service-name: calling handler -PUT -/test-client.0-2txq2dyjghs0vdf-335 - -host:smithi196.front.sepia.ceph.com -x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -x-amz-date:20220929T065029Z - -host;x-amz-content-sha256;x-amz-date -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -botocore.auth: DEBUG: StringToSign: -AWS4-HMAC-SHA256 -20220929T065029Z -20220929/us-east-1/s3/aws4_request -ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb -botocore.auth: DEBUG: Signature: -8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 -botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler -botocore.endpoint: DEBUG: Sending http request: -urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 -urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 -botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} -botocore.parsers: DEBUG: Response body: -b'' -botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler -botocore.retryhandler: DEBUG: No retry needed. -GET -/test-client.0-2txq2dyjghs0vdf-335 -tagging= -host:smithi196.front.sepia.ceph.com -x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -x-amz-date:20220929T065029Z - -host;x-amz-content-sha256;x-amz-date -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -botocore.auth: DEBUG: StringToSign: -AWS4-HMAC-SHA256 -20220929T065029Z -20220929/us-east-1/s3/aws4_request -8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 -botocore.auth: DEBUG: Signature: -a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 -botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler -botocore.endpoint: DEBUG: Sending http request: -urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 -urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 -botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} -botocore.parsers: DEBUG: Response body: -b'NoSuchTagSetErrortest-client.0-2txq2dyjghs0vdf-335tx00000ebc589e4bcad8d86-0063354035-1157-default1157-default-default' -botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler -botocore.retryhandler: DEBUG: No retry needed. -botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler > ---------------------- >> end captured logging << ---------------------]]> - \ No newline at end of file diff --git a/teuthology/util/test/files/test_unit_test.xml b/teuthology/util/test/files/test_unit_test.xml new file mode 100644 index 0000000000..bd9c73434c --- /dev/null +++ b/teuthology/util/test/files/test_unit_test.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/teuthology/util/test/files/test_valgrind.xml b/teuthology/util/test/files/test_valgrind.xml new file mode 100644 index 0000000000..41bf8375fd --- /dev/null +++ b/teuthology/util/test/files/test_valgrind.xml @@ -0,0 +1,31 @@ + + + + 0x870fc + 1 + Leak_DefinitelyLost + + 1,234 bytes in 1 blocks are definitely lost in loss record 198 of 201 + 1234 + 1 + + + + 0x4C39B6F + /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so + operator new[](unsigned long) + /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc + vg_replace_malloc.c + 640 + + + 0xF3F4B5 + /usr/bin/ceph-osd + ceph::common::leak_some_memory() + /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common + ceph_context.cc + 510 + + + + diff --git a/teuthology/util/test/test_scanner.py b/teuthology/util/test/test_scanner.py new file mode 100644 index 0000000000..1c7c89faa5 --- /dev/null +++ b/teuthology/util/test/test_scanner.py @@ -0,0 +1,191 @@ +from mock import patch, MagicMock + +from io import BytesIO +import os, io + +from teuthology.orchestra import remote +from teuthology.util.scanner import UnitTestScanner, ValgrindScanner + + +class MockFile(io.StringIO): + def close(self): + pass + + +class TestUnitTestScanner(object): + + def setup_method(self): + self.remote = remote.Remote( + name='jdoe@xyzzy.example.com', ssh=MagicMock()) + self.test_values = { + "xml_path": os.path.dirname(__file__) + "/files/test_unit_test.xml", + "error_msg": "FAILURE: Test `test_set_bucket_tagging` of `s3tests_boto3.functional.test_s3`. \ +Reason: 'NoSuchTagSetError' != 'NoSuchTagSet'.", + "summary_data": [{'failed_testsuites': {'s3tests_boto3.functional.test_s3': + [{'kind': 'failure', 'testcase': 'test_set_bucket_tagging', + 'message': "'NoSuchTagSetError' != 'NoSuchTagSet'"}]}, + 'num_of_failures': 1, + 'file_path': f'{os.path.dirname(__file__)}/files/test_unit_test.xml'}], + "yaml_data": r"""- failed_testsuites: + s3tests_boto3.functional.test_s3: + - kind: failure + message: '''NoSuchTagSetError'' != ''NoSuchTagSet''' + testcase: test_set_bucket_tagging + file_path: {file_dir}/files/test_unit_test.xml + num_of_failures: 1 +""".format(file_dir=os.path.dirname(__file__)) + } + + @patch('teuthology.util.scanner.UnitTestScanner.write_summary') + def test_scan_and_write(self, m_write_summary): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + result = UnitTestScanner(remote=self.remote).scan_and_write(xml_path, "test_summary.yaml") + assert result == self.test_values["error_msg"] + + def test_parse(self): + xml_content = b'\n\n\n' + scanner = UnitTestScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'FAILURE: Test `abc` of `xyz`. Reason: error_msg.', + {'failed_testsuites': {'xyz': + [{'kind': 'failure','message': 'error_msg','testcase': 'abc'}]}, + 'num_of_failures': 1 + } + ) + + def test_scan_file(self): + xml_path = self.test_values["xml_path"] + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = UnitTestScanner(remote=self.remote) + result = scanner.scan_file(xml_path) + assert result == self.test_values["error_msg"] + assert scanner.summary_data == self.test_values["summary_data"] + + def test_scan_all_files(self): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = UnitTestScanner(remote=self.remote) + result = scanner.scan_all_files(xml_path) + assert result == [self.test_values["error_msg"]] + + @patch('builtins.open') + def test_write_summary(self, m_open): + scanner = UnitTestScanner(self.remote) + mock_yaml_file = MockFile() + scanner.summary_data = self.test_values["summary_data"] + m_open.return_value = mock_yaml_file + scanner.write_summary("path/file.yaml") + written_content = mock_yaml_file.getvalue() + assert written_content == self.test_values["yaml_data"] + + +class TestValgrindScanner(object): + + def setup_method(self): + self.remote = remote.Remote( + name='jdoe@xyzzy.example.com', ssh=MagicMock()) + self.test_values = { + "xml_path": os.path.dirname(__file__) + "/files/test_valgrind.xml", + "error_msg": "valgrind error: Leak_DefinitelyLost\noperator new[]\ +(unsigned long)\nceph::common::leak_some_memory()", + "summary_data": [{'kind': 'Leak_DefinitelyLost', 'traceback': [{'file': + '/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c', + 'line': '640', 'function': 'operator new[](unsigned long)'}, + {'file': '/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc', + 'line': '510', 'function': 'ceph::common::leak_some_memory()'}], 'file_path': + f'{os.path.dirname(__file__)}/files/test_valgrind.xml'}], + "yaml_data": r"""- file_path: {file_dir}/files/test_valgrind.xml + kind: Leak_DefinitelyLost + traceback: + - file: /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c + function: operator new[](unsigned long) + line: '640' + - file: /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc + function: ceph::common::leak_some_memory() + line: '510' +""".format(file_dir=os.path.dirname(__file__)) + } + + def test_parse_with_traceback(self): + xml_content = b''' + + + Leak_DefinitelyLost + + + func() + /dir + file1.ext + 640 + + + + +''' + scanner = ValgrindScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'valgrind error: Leak_DefinitelyLost\nfunc()', + {'kind': 'Leak_DefinitelyLost', 'traceback': + [{'file': '/dir/file1.ext', 'line': '640', 'function': 'func()'}] + } + ) + + def test_parse_without_trackback(self): + xml_content = b''' + + + Leak_DefinitelyLost + + + + +''' + scanner = ValgrindScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'valgrind error: Leak_DefinitelyLost\n', + {'kind': 'Leak_DefinitelyLost', 'traceback': []} + ) + + def test_scan_file(self): + xml_path = self.test_values["xml_path"] + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = ValgrindScanner(remote=self.remote) + result = scanner.scan_file(xml_path) + assert result == self.test_values["error_msg"] + assert scanner.summary_data == self.test_values["summary_data"] + + def test_scan_all_files(self): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = ValgrindScanner(remote=self.remote) + result = scanner.scan_all_files(xml_path) + assert result == [self.test_values["error_msg"]] + + @patch('builtins.open') + def test_write_summary(self, m_open): + scanner = ValgrindScanner(self.remote) + mock_yaml_file = MockFile() + scanner.summary_data = self.test_values["summary_data"] + m_open.return_value = mock_yaml_file + scanner.write_summary("path/file.yaml") + written_content = mock_yaml_file.getvalue() + assert written_content == self.test_values["yaml_data"] \ No newline at end of file From fec6e4734bceefbf460ca2edf31751f8cbb08b81 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Fri, 27 Oct 2023 14:28:18 +0530 Subject: [PATCH 3/3] util/scanner: add UnitTestScanner.num_of_total_failures In UnitTestScanner's final error message, add total count of failures before the first error occurance, like "(total x failed) ". Another minor change: add "..." if the failure reason is more than 200 chars. Signed-off-by: Vallari Agrawal --- teuthology/util/scanner.py | 16 +++++++++++++--- teuthology/util/test/test_scanner.py | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/teuthology/util/scanner.py b/teuthology/util/scanner.py index 1e140d618c..b67d88c925 100644 --- a/teuthology/util/scanner.py +++ b/teuthology/util/scanner.py @@ -89,7 +89,7 @@ def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: if child.tag in ['failure', 'error']: fault_kind = child.tag reason = child.get('message', 'No message found in xml output, check logs.') - short_reason = reason[:200] + short_reason = (reason[:200].strip() + '...') if len(reason) > 200 else reason.strip() error_data[testcase_suitename] += [{ "kind": fault_kind, "testcase": testcase_name, @@ -99,7 +99,16 @@ def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.' return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) } - + + @property + def num_of_total_failures(self): + total_failed_testcases = 0 + if self.summary_data: + for file_data in self.summary_data: + failed_tests = file_data.get("num_of_failures", 0) + total_failed_testcases += failed_tests + return total_failed_testcases + def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]: """ Scan all files matching 'path_regex' @@ -109,7 +118,8 @@ def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]: errors = self.scan_all_files(path_regex) self.write_summary(summary_path) if errors: - return errors[0] + count = self.num_of_total_failures + return f"(total {count} failed) " + errors[0] except Exception as scanner_exc: log.error(str(scanner_exc)) diff --git a/teuthology/util/test/test_scanner.py b/teuthology/util/test/test_scanner.py index 1c7c89faa5..928d4305b7 100644 --- a/teuthology/util/test/test_scanner.py +++ b/teuthology/util/test/test_scanner.py @@ -44,7 +44,7 @@ def test_scan_and_write(self, m_write_summary): m_open.return_value = open(xml_path, "rb") self.remote._sftp_open_file = m_open result = UnitTestScanner(remote=self.remote).scan_and_write(xml_path, "test_summary.yaml") - assert result == self.test_values["error_msg"] + assert result == "(total 1 failed) " + self.test_values["error_msg"] def test_parse(self): xml_content = b'\n