From d10a36f5d2db8ccd8c426ccec2a34b47470fd128 Mon Sep 17 00:00:00 2001 From: "Rama Sasthri, Kristipati" Date: Tue, 15 Aug 2023 04:32:12 +0530 Subject: [PATCH] spytest pre-commit warnings fix in framework and use caching (#9430) Co-authored-by: Rama Sasthri, Kristipati --- spytest/bin/.pylintrc | 0 spytest/datastore/prompts/Prompts_Support.md | 11 ++-- spytest/spytest/batch.py | 35 +++++++++--- spytest/spytest/cmdargs.py | 6 +- spytest/spytest/ordyaml.py | 58 ++++++++++++++------ spytest/spytest/template.py | 36 +++++++----- spytest/spytest/testbed.py | 8 ++- spytest/spytest/tgen/tgen_utils.py | 8 +++ spytest/spytest/tgen_api.py | 13 +++++ spytest/tests/batch/test_spytest_infra_1.py | 5 ++ 10 files changed, 133 insertions(+), 47 deletions(-) delete mode 100644 spytest/bin/.pylintrc diff --git a/spytest/bin/.pylintrc b/spytest/bin/.pylintrc deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/spytest/datastore/prompts/Prompts_Support.md b/spytest/datastore/prompts/Prompts_Support.md index fd16f214b94..824c8183618 100644 --- a/spytest/datastore/prompts/Prompts_Support.md +++ b/spytest/datastore/prompts/Prompts_Support.md @@ -1,9 +1,9 @@ -# Prompts Support: - -We have placed a file "sonic_prompts.yaml" under "datastore/prompts" directory in spytest clone. +# Prompts Support: + +We have placed a file "sonic_prompts.yaml" under "datastore/prompts" directory in spytest clone. Little bit of help/comments are also provided in that file. -This file contains 3 sections: +This file contains 3 sections: #### **patterns:** Where users will define each pattern with a unique name. Each pattern name and value should be unique. For sonic management framework, "**--sonic-mgmt--**" is a default value used by framework, @@ -20,7 +20,7 @@ In the above section, for some commands, we added place holders. Names for those We have written an example script "**tests/infra_ut/test_ut_modes.py**" as part of our unit testing. Look for functions which match "**test_mgmt_cli_mode_\***", "**test_vtysh_prompt_modes_\***", "**test_vtysh_mgmt_prompt_modes_\***" and "**test_all_modes_\***" -# Example: +# Example: To add support for acl prompts, following is the way: @@ -35,4 +35,3 @@ And in modes section: For entering into that mode, ACL_NAME should be provided as input. So, need to add the following in required args **`mgmt-ipv4-acl-config: ['aclname']`** - diff --git a/spytest/spytest/batch.py b/spytest/spytest/batch.py index 7892ae0b538..23c4be09894 100644 --- a/spytest/spytest/batch.py +++ b/spytest/spytest/batch.py @@ -23,6 +23,9 @@ from spytest.version import get_git_ver import utilities.common as utils import utilities.parallel as putils +from utilities.profile import cprofile_start +from utilities.profile import cprofile_stop +from utilities.profile import dbg_cache # from utilities.tracer import Tracer @@ -38,6 +41,7 @@ def batch_init_env(wa): def batch_init(): # Tracer.register(trace_calls, "batch", include=os.path.abspath(__file__)) wa = SpyTestDict() + wa.start_time = get_timenow() wa.j2dict = SpyTestDict() wa.context = None wa.get_gw_name = {} @@ -153,15 +157,22 @@ def is_deadnode_recovery(): def ftrace(msg): + if env.match("SPYTEST_LOGS_TIME_FMT_ELAPSED", "1", "0"): + prefix = "{}: ".format(get_elapsed(wa.start_time, True)) + else: + prefix = "{}: ".format(get_timestamp()) + # prefix = "{} {}".format(get_worker_id(), prefix) + # prefix = "{} {}".format(os.getpid(), prefix) + if msg: + msg = utils.augment_lines(msg, prefix) if wa.logs_path: if not wa.trace_file: wa.trace_file = os.path.join(wa.logs_path, "batch_debug.log") utils.write_file(wa.trace_file, "") if msg: - prefix = "{}: ".format(get_timestamp()) - # prefix = "{} {}".format(get_worker_id(), prefix) - # prefix = "{} {}".format(os.getpid(), prefix) - utils.write_file(wa.trace_file, "{}{}\n".format(prefix, msg), "a") + utils.write_file(wa.trace_file, msg, "a", "\n") + if msg and wa.debug_level >= 100: + print(msg) def debug(*args, **kwargs): @@ -220,7 +231,7 @@ def check_worker_status(node_modules, collection): if not worker.started: continue if worker.last_report: - elapsed = get_elapsed(worker.last_report, False) + elapsed = get_elapsed(worker.last_report) else: elapsed = 0 try: @@ -1206,14 +1217,16 @@ def _update_applicable(self): def _show_testbed_topo(show=True): - header = ["Node", "Topology"] - rows = [] + header, rows = ["Node", "Topology"], [] + lines = utils.banner("TOPOLOGY", func="").strip().split("\n") for worker in wa.workers.values(): topo = worker.tb_obj.get_topo() rows.append([worker.name, topo]) + lines.append("{:5s} = {}".format(worker.name, topo)) retval = utils.sprint_vtable(header, rows) if show: - trace(retval) + # trace(retval) + trace("\n".join(lines)) return retval @@ -2305,6 +2318,7 @@ def parse_buckets(count, testbeds, buckets_csv, logs_path): # create testbed objects for testbed files trace("============> Parsing Testbed files") + cprofile_start() for testbed in testbeds: # wa.logger = wa.logger or logging.getLogger() tb = Testbed(testbed, logger=wa.logger, flex_dut=True) @@ -2315,6 +2329,7 @@ def parse_buckets(count, testbeds, buckets_csv, logs_path): # trace("Testbed: {}".format(testbed)) # trace(" Devices: {}".format(tb.get_device_names("DUT"))) tb_objs.append(tb) + cprofile_stop(os.path.join(logs_path, "batch_profile_parse_testbed")) # initialize collected lists for key in ["testbeds", "buckets", "min_buckets", "parent_testbeds", "testbed_objs"]: @@ -2334,6 +2349,7 @@ def parse_buckets(count, testbeds, buckets_csv, logs_path): wa.parent_testbeds.append(parent_testbed) trace("============> Parse Bucket Testbed files") + cprofile_start() for testbed in wa.testbeds: fname = os.path.basename(testbed) tb = Testbed(testbed, logger=wa.logger, flex_dut=True) @@ -2344,6 +2360,9 @@ def parse_buckets(count, testbeds, buckets_csv, logs_path): msg = "Topology({}): {}" trace(msg.format(fname, tb.get_topo())) wa.testbed_objs.append(tb) + cprofile_stop(os.path.join(logs_path, "batch_profile_buckets")) + if wa.debug_level >= 100: + dbg_cache() wa.workers.clear() wa.testbed_count = len(wa.testbeds) diff --git a/spytest/spytest/cmdargs.py b/spytest/spytest/cmdargs.py index 97e49d90b1f..f06debcf0bc 100644 --- a/spytest/spytest/cmdargs.py +++ b/spytest/spytest/cmdargs.py @@ -303,13 +303,13 @@ class HelpFormatter(argparse.HelpFormatter): def _format_action(self, action): if type(action).__name__ in ["ArgValidateEnv", "ArgValidateMaxTime"]: action.nargs = 1 - if type(action) == argparse._SubParsersAction: + if isinstance(action, argparse._SubParsersAction): # inject new class variable for sub command formatting subactions = action._get_subactions() invocations = [self._format_action_invocation(a) for a in subactions] self._subcommand_max_length = max(len(i) for i in invocations) - if type(action) == argparse._SubParsersAction._ChoicesPseudoAction: + if isinstance(action, argparse._SubParsersAction._ChoicesPseudoAction): # format sub command help line subcommand = self._format_action_invocation(action) # type: str width = self._subcommand_max_length @@ -318,7 +318,7 @@ def _format_action(self, action): help_text = self._expand_help(action) return " {:{width}} - {}\n".format(subcommand, help_text, width=width) - elif type(action) == argparse._SubParsersAction: + elif isinstance(action, argparse._SubParsersAction): # process sub command help section msg = '\n' for subaction in action._get_subactions(): diff --git a/spytest/spytest/ordyaml.py b/spytest/spytest/ordyaml.py index 489aaec193c..bda1ac69d27 100644 --- a/spytest/spytest/ordyaml.py +++ b/spytest/spytest/ordyaml.py @@ -1,9 +1,11 @@ +import re import os import copy import yaml from spytest.dicts import SpyTestDict import utilities.common as utils +from utilities.profile import get_cache, set_cache class NoAliasDumper(yaml.SafeDumper): @@ -25,17 +27,23 @@ def _locate(self, filename): def _load(self, stream, file_dict=dict(), Loader=yaml.Loader, object_pairs_hook=SpyTestDict): def _yaml_include(loader, node): - filename = self._locate(node.value) - if not filename: + file_path = self._locate(node.value) + if not file_path: msg = "Failed to locate included file '{}'".format(node.value) self.errs.append(msg) return None - file_dict[filename] = 1 - with utils.open_file(filename) as inputfile: - rv = yaml.load(inputfile, Loader) + file_dict[file_path] = 1 + rv = get_cache("ordyaml.include", file_path, None) + if rv: if not self.expand_include: - return self._add_include_map(node, rv) + rv = self._add_include_map(node, rv) return rv + text = self.read_file(file_path) + rv = yaml.load(text, Loader) + set_cache("ordyaml.include", file_path, rv) + if not self.expand_include: + rv = self._add_include_map(node, rv) + return rv def _construct_mapping(loader, node): loader.flatten_mapping(node) @@ -106,32 +114,50 @@ def __init__(self, filename, paths=[], content=""): else: self.init_content(content) - def init_content(self, content): + def init_content(self, content, for_file=None): all_files = dict() try: self.text0 = content - self.obj = self._load(self.text0, all_files, yaml.SafeLoader) - self.text1 = self._dump(self.obj) + rv1 = get_cache("ordyaml.init_content.load", for_file, None) if for_file else None + rv2 = get_cache("ordyaml.init_content.dump", for_file, None) if for_file else None + if None not in [rv1, rv2]: + self.obj = copy.deepcopy(rv1) + self.text1 = copy.deepcopy(rv2) + else: + self.obj = self._load(self.text0, all_files, yaml.SafeLoader) + self.text1 = self._dump(self.obj) + set_cache("ordyaml.init_content.load", for_file, self.obj) + set_cache("ordyaml.init_content.dump", for_file, self.text1) self.valid = True return all_files except Exception as e: self.errs.append(e) raise (e) + def read_file(self, file_path): + fh = utils.open_file(file_path) + if not fh: + return None + text = fh.read() + text = re.sub(r"[\"|']!include (.*).yaml[\"|']", r"!include \1.yaml", text) + fh.close() + return text + def init_file(self, filename, paths=[]): self._init_paths(filename, paths) file_path = self._locate(filename) if not file_path: self.errs.append("File {} not found".format(filename)) return None - fh = utils.open_file(file_path) - if not fh: - self.errs.append("Failed to open {}".format(filename)) - return None try: - text0 = fh.read() - fh.close() - self.all_files = self.init_content(text0) + text = get_cache("ordyaml.init", file_path, None) + if text is None: + text = self.read_file(file_path) + if text is None: + self.errs.append("Failed to open {}".format(filename)) + return None + set_cache("ordyaml.init", file_path, text) + self.all_files = self.init_content(text, file_path) self.all_files[file_path] = 1 return file_path except Exception as e: diff --git a/spytest/spytest/template.py b/spytest/spytest/template.py index 1a3c8f992e2..411cfa85c5d 100644 --- a/spytest/spytest/template.py +++ b/spytest/spytest/template.py @@ -9,14 +9,14 @@ vendor_dir = os.path.join(os.path.dirname(__file__), '..', "vendor") sys.path.insert(0, os.path.abspath(vendor_dir)) -import textfsm # noqa: E402 +import textfsm # noqa: E402 try: import clitable except Exception: from textfsm import clitable -from spytest import env # noqa: E402 -import utilities.common as utils # noqa: E402 +from spytest import env # noqa: E402 +import utilities.common as utils # noqa: E402 class Template(object): @@ -51,6 +51,14 @@ def get_tmpl(self, cmd): return cli_table.index.index[row_idx]['Template'] return None + def get_table(self, cmd): + attrs = dict(Command=cmd) + for cli_table in self.cli_tables.values(): + row_idx = cli_table.index.GetRowMatch(attrs) + if row_idx != 0: + return cli_table + return None + # retrieve template and sample file given the command def read_sample(self, cmd): tmpl_file = self.get_tmpl(cmd) @@ -81,16 +89,18 @@ def apply(self, output, cmd): attrs["Platform"] = self.platform if self.cli: attrs["cli"] = self.cli - try: - tmpl_file = self.get_tmpl(cmd) - if not tmpl_file: - raise ValueError('Unknown command "%s"' % (cmd)) - for cli_table in self.cli_tables.values(): - cli_table.ParseCmd(output, attrs) - objs = self.result(cli_table.header, cli_table) - return [tmpl_file, objs] - except clitable.CliTableError as e: - raise ValueError('Unable to parse command "%s" - %s' % (cmd, str(e))) + + tmpl_file = self.get_tmpl(cmd) + if not tmpl_file: + raise ValueError('Unknown command "%s"' % (cmd)) + + cli_table = self.get_table(cmd) + if not cli_table: + raise ValueError('Unable to parse command "%s"' % (cmd)) + + cli_table.ParseCmd(output, attrs) + objs = self.result(cli_table.header, cli_table) + return [tmpl_file, objs] def result(self, header, rows): objs = [] diff --git a/spytest/spytest/testbed.py b/spytest/spytest/testbed.py index 7272ecee285..fd8d36b7e10 100644 --- a/spytest/spytest/testbed.py +++ b/spytest/spytest/testbed.py @@ -16,6 +16,7 @@ from spytest.st_time import get_elapsed import utilities.common as utils +from utilities.profile import get_cache, set_cache testbeds_root = os.path.join(os.path.dirname(__file__), '..') testbeds_root = os.path.join(os.path.abspath(testbeds_root), "testbeds") @@ -733,10 +734,15 @@ def _load_yaml(self, filename): errs = [] try: user_root = env.get("SPYTEST_USER_ROOT") - if user_root: + rv = get_cache("testbed.load.yaml", filename, None) + if rv: + self.oyaml = copy.deepcopy(rv) + elif user_root: self.oyaml = OrderedYaml(filename, [user_root, testbeds_root]) + set_cache("testbed.load.yaml", filename, self.oyaml) else: self.oyaml = OrderedYaml(filename, [testbeds_root]) + set_cache("testbed.load.yaml", filename, self.oyaml) if not self.oyaml.is_valid(): errs = self.oyaml.get_errors() self.logger.error(errs) diff --git a/spytest/spytest/tgen/tgen_utils.py b/spytest/spytest/tgen/tgen_utils.py index 7d939c9f96b..1ecfda7ce55 100644 --- a/spytest/spytest/tgen/tgen_utils.py +++ b/spytest/spytest/tgen/tgen_utils.py @@ -5,6 +5,12 @@ from spytest.tgen_api import get_chassis from spytest.tgen_api import is_soft_tgen +_latest_log_msg = "" + + +def get_latest_log_msg(): + return _latest_log_msg + def _log_call(fname, **kwargs): args_list = [] @@ -238,6 +244,8 @@ def _log_validation(cmsg, result, exp_val, real_rx_val, diff, strelem=None, fpel if fpelem: msg = msg + " filter param: {} value: {}".format(fpelem, fvelem) st.log(msg) + global _latest_log_msg + _latest_log_msg = msg def _verify_aggregate_stats(tr_details, **kwargs): diff --git a/spytest/spytest/tgen_api.py b/spytest/spytest/tgen_api.py index c0142838fc9..2b3120ab625 100644 --- a/spytest/spytest/tgen_api.py +++ b/spytest/spytest/tgen_api.py @@ -4,6 +4,11 @@ def get_counter_name(mode, tg_type, comp_type, direction): return tgen_utils.get_counter_name(mode, tg_type, comp_type, direction) +def get_latest_log_msg(): + from spytest.tgen import tgen_utils + return tgen_utils.get_latest_log_msg() + + def validate_tgen_traffic(**kwargs): from spytest.tgen import tgen_utils return tgen_utils.validate_tgen_traffic(**kwargs) @@ -177,6 +182,14 @@ def send_verify_traffic(tg, stream, ph_rx, ph_tx, tolerance, stop_time=5, run_ti return None +def report_traffic_verification_fail(msgid=None, line=None): + from spytest import st + from utilities.common import get_line_number + msgid = msgid or "failed_traffic_verification" + line = line or get_line_number(1) + st.report_fail(msgid, "{} @{}".format(get_latest_log_msg(), line)) + + def get_min(v1, v2): return v1 if v1 < v2 else v2 diff --git a/spytest/tests/batch/test_spytest_infra_1.py b/spytest/tests/batch/test_spytest_infra_1.py index 0da9f3099d0..28d24f57b55 100644 --- a/spytest/tests/batch/test_spytest_infra_1.py +++ b/spytest/tests/batch/test_spytest_infra_1.py @@ -1,8 +1,13 @@ from spytest import st + def test_spytest_infra_first(): st.report_pass("test_case_passed") + def test_spytest_infra_second(): st.report_pass("test_case_passed") + +def test_spytest_infra_last(): + st.report_pass("test_case_passed")