diff --git a/bin/ovs-lgrep b/bin/ovs-lgrep index bc18588..2867bc7 100755 --- a/bin/ovs-lgrep +++ b/bin/ovs-lgrep @@ -18,14 +18,14 @@ from ovs_dbg.logs import OVSLog @click.option( "-s", "--start", - help="Start timestamp." + help="Start timestamp. " "Format (same as OVS logs): '%Y-%m-%dT%H:%M:%S.%fZ'" ", e.g: '2021-07-15T15:04:05.793Z'", ) @click.option( "-e", "--end", - help="End timestamp" + help="End timestamp. " "Format (same as OVS logs): '%Y-%m-%dT%H:%M:%S.%fZ'" ", e.g: '2021-07-15T15:04:05.793Z'", ) diff --git a/docs/source/index.rst b/docs/source/index.rst index 343fc60..289335d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,8 @@ This respository stores scripts and documentation to help debug OVS and OVN. ofofproto + lgrep + Indices and tables ================== diff --git a/docs/source/lgrep.rst b/docs/source/lgrep.rst new file mode 100644 index 0000000..a700261 --- /dev/null +++ b/docs/source/lgrep.rst @@ -0,0 +1,81 @@ +================================== +ovs-lgrep: Time-aware log grepping +================================== + +Often, when troubleshooting OVS or OVN problems, we need to correlate +multiple log files (from multiple nodes, OVN, etc). To make that task +a bit easer `ovs-lgrep` allows you to look for logs at a specific point +in time in many log files. + + +----- +Usage +----- + +:: + + ovs-lgrep -h + + +Search logs by regular expression +********************************* + +ovs-lgrep uses Python's regular expression syntax, see the regexp_syntax_. +Grep for a specific expression by running: + + +:: + + ovs-lgrep -r "peak resident set size grew" ovn-northd.log + + +.. _regexp_syntax: https://docs.python.org/3/library/re.html + + +Search by timestamp +******************* + +Start and end boundaries can be specified: + +:: + + ovs-lgrep -s "2021-07-15T16:50:03.018Z" -e "2021-07-15T17:42:03.492Z" + ovn-northd.log + + +Time boundaries can be specified using `-A` and `-B` options: + +:: + + ovs-lgrep -s "2021-07-15T16:50:03.018Z" -A 2m ovn-northd.log + ovs-lgrep -s "2021-07-15T16:50:03.018Z" -B 1h2m3s ovn-northd.log + + +Logfile interleaving +******************** + +If multiple log files are specified, the result of the search will be interleaved +to help analyze the distributed system. + +:: + + ovs-lgrep -s "2021-07-15T16:50:03.018Z" -A 2m */ovn-northd.log + + --- File: ovn-central-1/ovn-northd.log --- + 2021-07-15T16:50:03.018Z|00252|poll_loop|INFO|Dropped 4 log messages in last 0 + seconds (most recently, 0 seconds ago) due to excessive rate + 2021-07-15T16:50:03.018Z|00253|poll_loop|INFO|wakeup due to [POLLIN] on fd 12 + (192.16.0.1:46952<->192.16.0.1:6641) at lib/stream-ssl.c:832 (91% CPU usage) + 2021-07-15T16:50:03.589Z|00254|ovsdb_cs|INFO|ssl:192.16.0.3:6642: clustered + database server is not cluster leader; trying another server + 2021-07-15T16:50:03.589Z|00255|ovn_northd|INFO|ovn-northd lock lost. This + ovn-northd instance is now on standby. + --- File: ovn-central-2/ovn-northd.log --- + 2021-07-15T16:50:03.590Z|00057|ovsdb_cs|INFO|ssl:192.16.0.3:6642: clustered + database server is not cluster leader; trying another server + --- File: ovn-central-3/ovn-northd.log --- + 2021-07-15T16:50:03.590Z|00057|ovsdb_cs|INFO|ssl:192.16.0.3:6642: clustered + database server is not cluster leader; trying another server + --- File: ovn-central-1/ovn-northd.log --- + 2021-07-15T16:50:11.597Z|00256|reconnect|INFO|ssl:192.16.0.1:6642: connected + diff --git a/ovs_dbg/decoders.py b/ovs_dbg/decoders.py index 1a38c91..5412a1f 100644 --- a/ovs_dbg/decoders.py +++ b/ovs_dbg/decoders.py @@ -115,7 +115,7 @@ def max(self): return (self.max_mask() & ~self._mask) | (self._value & self._mask) def __str__(self): - if self.fully: + if self.fully(): return str(self._value) else: return "{}/{}".format(hex(self._value), hex(self._mask)) diff --git a/ovs_dbg/flow.py b/ovs_dbg/flow.py index 40b4604..e2f5954 100644 --- a/ovs_dbg/flow.py +++ b/ovs_dbg/flow.py @@ -50,19 +50,26 @@ class Flow(object): Args: sections (list[Section]): list of sections that comprise the flow orig (str): Original flow string + id (Any): Identifier """ - def __init__(self, sections, orig=""): + def __init__(self, sections, orig="", id=None): self._sections = sections self._orig = orig + self._id = id for section in sections: - setattr( self, section.name, self.section(section.name).format_data()) + setattr(self, section.name, self.section(section.name).format_data()) setattr(self, "{}_kv".format(section.name), self.section(section.name).data) def section(self, name): """Return the section by name""" return next((sect for sect in self._sections if sect.name == name), None) + @property + def id(self): + """Return the Flow ID""" + return self._id + @property def sections(self): """Return the section by name""" diff --git a/ovs_dbg/odp.py b/ovs_dbg/odp.py index 100831b..700415d 100644 --- a/ovs_dbg/odp.py +++ b/ovs_dbg/odp.py @@ -33,12 +33,12 @@ class ODPFlow(Flow): """Datapath Flow""" - def __init__(self, sections, raw=""): + def __init__(self, sections, raw="", id=None): """Constructor""" - super(ODPFlow, self).__init__(sections, raw) + super(ODPFlow, self).__init__(sections, raw, id) @classmethod - def from_string(cls, odp_string): + def from_string(cls, odp_string, id=None): """Parse a odp flow string The string is expected to have the follwoing format: @@ -109,7 +109,7 @@ def from_string(cls, odp_string): ) sections.append(asection) - return cls(sections, odp_string) + return cls(sections, odp_string, id) @classmethod def _action_parser(cls): diff --git a/ovs_dbg/ofp.py b/ovs_dbg/ofp.py index 35422a0..01d81dc 100644 --- a/ovs_dbg/ofp.py +++ b/ovs_dbg/ofp.py @@ -41,12 +41,12 @@ class OFPFlow(Flow): """OpenFlow Flow""" - def __init__(self, sections, orig=""): + def __init__(self, sections, orig="", id=None): """Constructor""" - super(OFPFlow, self).__init__(sections, orig) + super(OFPFlow, self).__init__(sections, orig, id) @classmethod - def from_string(cls, ofp_string): + def from_string(cls, ofp_string, id=None): """Parse a ofproto flow string The string is expected to have the follwoing format: @@ -102,7 +102,7 @@ def from_string(cls, ofp_string): ) sections.append(asection) - return cls(sections, ofp_string) + return cls(sections, ofp_string, id) @classmethod def _info_decoders(cls): diff --git a/ovs_dbg/ofparse/console.py b/ovs_dbg/ofparse/console.py index 76a33cc..06de0b9 100644 --- a/ovs_dbg/ofparse/console.py +++ b/ovs_dbg/ofparse/console.py @@ -1,231 +1,146 @@ -""" This module defines OFConsole class +""" This module defines console formatting """ -import sys +import colorsys import contextlib +import itertools +import sys +import zlib from rich.console import Console from rich.text import Text from rich.style import Style +from rich.color import Color + +from ovs_dbg.ofparse.format import FlowFormatter, FlowBuffer, FlowStyle -class OFConsole: - """OFConsole is a class capable of printing flows in a rich console format +class ConsoleBuffer(FlowBuffer): + """ConsoleBuffer implements FlowBuffer to provide console-based text + formatting based on rich.Text + + Append functions accept a rich.Style Args: - console (rich.Console): Optional, an existing console to use - max_value_len (int): Optional; max length of the printed values - kwargs (dict): Optional; Extra arguments to be passed down to - rich.console.Console() + rtext(rich.Text): Optional; text instance to reuse """ - default_style = { - "key": Style(color="steel_blue"), - "delim": Style(color="steel_blue"), - "value": Style(color="medium_orchid"), - "value.type.IPAddress": Style(color="green4"), - "value.type.IPMask": Style(color="green4"), - "value.type.EthMask": Style(color="green4"), - "value.ct": Style(color="bright_black"), - "value.ufid": Style(color="dark_red"), - "value.clone": Style(color="bright_black"), - "value.controller": Style(color="bright_black"), - "flag": Style(color="slate_blue1"), - "key.drop": Style(color="red"), - "key.resubmit": Style(color="green3"), - "key.output": Style(color="green3"), - } - - def __init__(self, console=None, max_value_length=-1, **kwargs): - self.console = console or Console(**kwargs) - self.max_value_length = max_value_length - - def print_flow(self, flow, style=None): - """ - Prints a flow to the console + def __init__(self, rtext): + self._text = rtext or Text() - Args: - flow (ovs_dbg.OFPFlow): the flow to print - style (dict): Optional; style dictionary to use - """ + @property + def text(self): + return self._text - text = Text() - self.format_flow(flow, style, text) - self.console.print(text) + def _append(self, string, style): + """Append to internal text""" + return self._text.append(string, style) - def format_flow(self, flow, style=None, text=None): + def append_key(self, kv, style): + """Append a key + Args: + kv (KeyValue): the KeyValue instance to append + style (rich.Style): the style to use """ - Formats the flow into the rich.Text + return self._append(kv.meta.kstring, style) + def append_delim(self, kv, style): + """Append a delimiter Args: - flow (ovs_dbg.OFPFlow): the flow to format - style (dict): Optional; style dictionary to use - text (rich.Text): Optional; the Text object to append to - """ - text = text if text is not None else Text() - - last_printed_pos = 0 - for section in sorted(flow.sections, key=lambda x: x.pos): - text.append( - flow.orig[last_printed_pos : section.pos], - Style(color="white"), - ) - self.format_kv_list(section.data, section.string, style, text) - last_printed_pos = section.pos + len(section.string) - - def format_info(self, flow, style=None, text=None): + kv (KeyValue): the KeyValue instance to append + style (rich.Style): the style to use """ - Formats the flow information into the rich.Text + return self._append(kv.meta.delim, style) + def append_end_delim(self, kv, style): + """Append an end delimiter Args: - flow (ovs_dbg.OFPFlow): the flow to format - style (dict): Optional; style dictionary to use - text (rich.Text): Optional; the Text object to append to + kv (KeyValue): the KeyValue instance to append + style (rich.Style): the style to use """ - self.format_kv_list(flow.info_kv, flow.meta.istring, style, text) + return self._append(kv.meta.end_delim, style) - def format_matches(self, flow, style=None, text=None): + def append_value(self, kv, style): + """Append a value + Args: + kv (KeyValue): the KeyValue instance to append + style (rich.Style): the style to use """ - Formats the flow information into the rich.Text + return self._append(kv.meta.vstring, style) + def append_extra(self, extra, style): + """Append extra string Args: - flow (ovs_dbg.OFPFlow): the flow to format - style (dict): Optional; style dictionary to use - text (rich.Text): Optional; the Text object to append to + kv (KeyValue): the KeyValue instance to append + style (rich.Style): the style to use """ - self.format_kv_list(flow.match_kv, flow.meta.mstring, style, text) + return self._append(extra, style) + - def format_actions(self, flow, style=None, text=None): +class ConsoleFormatter(FlowFormatter): + """ + Args: + console (rich.Console): Optional, an existing console to use + max_value_len (int): Optional; max length of the printed values + kwargs (dict): Optional; Extra arguments to be passed down to + rich.console.Console() + """ + + def __init__(self, opts=None, console=None, **kwargs): + super(ConsoleFormatter, self).__init__() + style = self.style_from_opts(opts) + self.console = console or Console(no_color=(style is None), **kwargs) + self.style = style or FlowStyle() + + def style_from_opts(self, opts): + return self._style_from_opts(opts, "console", Style) + + def print_flow(self, flow): """ - Formats the action into the rich.Text + Prints a flow to the console Args: - flow (ovs_dbg.OFPFlow): the flow to format + flow (ovs_dbg.OFPFlow): the flow to print style (dict): Optional; style dictionary to use - text (rich.Text): Optional; the Text object to append to """ - self.format_kv_list(flow.actions_kv, flow.meta.astring, style, text) - def format_kv_list(self, kv_list, full_str, style=None, text=None): + buf = ConsoleBuffer(Text()) + self.format_flow(buf, flow) + self.console.print(buf.text) + + def format_flow(self, buf, flow): """ - Formats the list of KeyValues into the rich.Text + Formats the flow into the provided buffer as a rich.Text Args: - kv_list (list[KeyValue]): the flow to format - full_str (str): the full string containing all k-v - style (dict): Optional; style dictionary to use - text (rich.Text): Optional; the Text object to append to + buf (FlowBuffer): the flow buffer to append to + flow (ovs_dbg.OFPFlow): the flow to format + style (FlowStyle): Optional; style object to use """ - text = text if text is not None else Text() - for i in range(len(kv_list)): - kv = kv_list[i] - written = self.format_kv(kv, style=style, text=text) - - # print kv separators - end = kv_list[i + 1].meta.kpos if i < (len(kv_list) - 1) else len(full_str) - text.append( - full_str[(kv.meta.kpos + written) : end].rstrip("\n\r"), - style=Style(color="white"), - ) - - def format_kv(self, kv, style=None, text=None, highlighted=[]): - """Format a KeyValue - - A formatted keyvalue has the following parts: - {key}{delim}{value}[{delim}] - - The following keys are fetched in style dictionary to determine the - style to use for the key section: - - key.highlighted.{key} (if key is found in hightlighted) - - key.highlighted (if key is found in hightlighted) - - key.{key} - - key - - The following keys are fetched in style dictionary to determine the - style to use for the value section of a specific key: - - value.highlighted.{key} (if key is found in hightlighted) - - value.highlighted.type{value.__class__.__name__} - - value.highlighted - (if key is found in hightlighted) - - value.{key} - - value.type.{value.__class__.__name__} - - value - - The following keys are fetched in style dictionary to determine the - style to use for the delim section - - delim + return super(ConsoleFormatter, self).format_flow(buf, flow, self.style) - Args: - kv (KeyValue): The KeyValue to print - text (rich.Text): Optional; Text instance to append the text to - style (dict): The style dictionary - highlighted(list): A list of keys that shall be highlighted - Returns the number of printed characters - """ - ret = 0 - text = text if text is not None else Text() - styles = style or self.default_style - meta = kv.meta - key = meta.kstring - - if kv.value is True and not kv.meta.vstring: - text.append(key, styles.get("flag")) - return len(key) - - key_style_lookup = ( - ["key.highlighted.%s" % key, "key.highlighted"] - if key in highlighted - else [] - ) - key_style_lookup.extend(["key.%s" % key, "key"]) - key_style = next(styles.get(s) for s in key_style_lookup if styles.get(s)) - - text.append(key, key_style) - ret += len(key) - - if kv.meta.vstring: - if kv.meta.delim not in ("\n", "\t", "\r", ""): - text.append(kv.meta.delim, styles.get("delim")) - ret += len(kv.meta.delim) - - value_style_lookup = ( - [ - "value.highlighted.%s" % key, - "value.highlighted.type.%s" % kv.value.__class__.__name__, - "value.highlighted", - ] - if key in highlighted - else [] - ) - value_style_lookup.extend( - [ - "value.%s" % key, - "value.type.%s" % kv.value.__class__.__name__, - "value", - ] - ) - value_style = next( - styles.get(s) for s in value_style_lookup if styles.get(s) - ) - - if ( - self.max_value_length >= 0 - and len(kv.meta.vstring) > self.max_value_length - ): - value_str = kv.meta.vstring[0 : self.max_value_length] + "..." - else: - value_str = kv.meta.vstring - - text.append(value_str, style=value_style) - ret += len(kv.meta.vstring) - if meta.end_delim: - text.append(meta.end_delim, styles.get("delim")) - ret += len(kv.meta.end_delim) - - return ret - - -def print_context(console, paged=False, styles=True): +def hash_pallete(hue, saturation, value): + """Generates a color pallete with the cartesian product + of the hsv values provided and returns a callable that assigns a color for + each value hash + """ + HSV_tuples = itertools.product(hue, saturation, value) + RGB_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples) + styles = [ + Style(color=Color.from_rgb(r * 255, g * 255, b * 255)) for r, g, b in RGB_tuples + ] + + def get_style(string): + hash_val = zlib.crc32(bytes(str(string), "utf-8")) + print(hash_val) + print(hash_val % len(styles)) + print(len(styles)) + return styles[hash_val % len(styles)] + + return get_style + + +def print_context(console, opts): """ Returns a printing context @@ -234,7 +149,7 @@ def print_context(console, paged=False, styles=True): paged (bool): Wheter to page the output style (bool): Whether to force the use of styled pager """ - if paged: + if opts.get("paged"): # Internally pydoc's pager library is used which returns a # plain pager if both stdin and stdout are not tty devices # @@ -243,6 +158,8 @@ def print_context(console, paged=False, styles=True): if not sys.stdin.isatty() and sys.stdout.isatty(): setattr(sys.stdin, "isatty", lambda: True) - return console.pager(styles=styles) + with_style = opts.get("style") is not None + + return console.pager(styles=with_style) return contextlib.nullcontext() diff --git a/ovs_dbg/ofparse/dp.py b/ovs_dbg/ofparse/dp.py index e0f16cf..402374a 100644 --- a/ovs_dbg/ofparse/dp.py +++ b/ovs_dbg/ofparse/dp.py @@ -1,6 +1,9 @@ import sys import click import colorsys +import graphviz +import itertools + from rich.tree import Tree from rich.text import Text from rich.console import Console @@ -9,8 +12,16 @@ from ovs_dbg.ofparse.main import maincli from ovs_dbg.ofparse.process import process_flows, tojson, pprint -from .console import OFConsole, print_context +from ovs_dbg.ofparse.console import ( + ConsoleFormatter, + ConsoleBuffer, + print_context, + hash_pallete, +) +from ovs_dbg.ofparse.format import FlowStyle +from ovs_dbg.ofparse.html import HTMLBuffer, HTMLFormatter from ovs_dbg.odp import ODPFlow +from ovs_dbg.filter import OFFilter @maincli.group(subcommand_metavar="FORMAT") @@ -52,51 +63,366 @@ def callback(flow): ) tree = Tree("Datapath Flows (logical)") - console = Console(color_system="256") - ofconsole = OFConsole(console) - - recirc_styles = [ - Style(color=Color.from_rgb(r * 255, g * 255, b * 255)) - for r, g, b in create_color_pallete(50) - ] - - def process_flow_tree(parent, recirc_id): - sorted_flows = sorted( - filter(lambda f: f.match.get("recirc_id") == recirc_id, flow_list), - key=lambda x: x.info.get("packets") or 0, - reverse=True, - ) + ofconsole = ConsoleFormatter(opts) + console = ofconsole.console - style = OFConsole.default_style - style["value"] = Style(color="bright_black") - style["key.output"] = Style(color="green") - style["value.output"] = Style(color="green") - for flow in sorted_flows: - next_recirc = next( - (kv.value for kv in flow.actions_kv if kv.key == "recirc"), None - ) - if next_recirc: - style["value.recirc"] = recirc_styles[next_recirc % len(recirc_styles)] + # HSV_tuples = [(x / size, 0.7, 0.8) for x in range(size)] + recirc_style_gen = hash_pallete( + hue=[x / 50 for x in range(0, 50)], saturation=[0.7], value=[0.8] + ) - text = Text() - style["value.recirc_id"] = recirc_styles[ - (flow.match.get("recirc_id")) % len(recirc_styles) - ] - ofconsole.format_flow(flow=flow, style=style, text=text) - tree_elem = parent.add(text) + style = ofconsole.style + style.set_default_value_style(Style(color="bright_black")) + style.set_key_style("output", Style(color="green")) + style.set_value_style("output", Style(color="green")) + style.set_value_style("recirc", recirc_style_gen) + style.set_value_style("recirc_id", recirc_style_gen) - if next_recirc: - process_flow_tree(tree_elem, next_recirc) + def append_to_tree(parent, flow): + buf = ConsoleBuffer(Text()) + ofconsole.format_flow(buf, flow) + tree_elem = parent.add(buf.text) + return tree_elem - process_flow_tree(tree, 0) + process_flow_tree(flow_list, tree, 0, append_to_tree) - with print_context(console, opts["paged"], not opts["no_style"]): + with print_context(console, opts): console.print(tree) -def create_color_pallete(size): - """Create a color pallete of size colors by modifying the Hue in the HSV - color space +@datapath.command() +@click.pass_obj +def html(opts): + """Print the flows in an HTML list sorted by recirc_id""" + + flow_list = [] + + def callback(flow): + flow_list.append(flow) + + process_flows( + flow_factory=ODPFlow.from_string, + callback=callback, + filename=opts.get("filename"), + filter=opts.get("filter"), + ) + + html_obj = get_html_obj(flow_list) + + print(html_obj) + + +@datapath.command() +@click.option( + "-h", + "--html", + is_flag=True, + default=False, + show_default=True, + help="Output an html file containing the graph", +) +@click.pass_obj +def graph(opts, html): + """Print the flows in an graphviz (.dot) format showing the relationship of recirc_ids""" + + recirc_flows = {} + + def callback(flow): + """Parse the flows and sort them by table""" + rid = hex(flow.match.get("recirc_id") or 0) + if not recirc_flows.get(rid): + recirc_flows[rid] = list() + recirc_flows[rid].append(flow) + + process_flows( + flow_factory=ODPFlow.from_string, + callback=callback, + filename=opts.get("filename"), + filter=opts.get("filter"), + ) + + node_styles = { + OFFilter("ct and (ct_state or ct_label or ct_mark)"): {"color": "#ff00ff"}, + OFFilter("ct_state or ct_label or ct_mark"): {"color": "#0000ff"}, + OFFilter("ct"): {"color": "#ff0000"}, + } + + g = graphviz.Digraph("DP flows", node_attr={"shape": "rectangle"}) + g.attr(compound="true") + g.attr(rankdir="TB") + + for recirc, flows in recirc_flows.items(): + with g.subgraph( + name="cluster_{}".format(recirc), comment="recirc {}".format(recirc) + ) as sg: + + sg.attr(rankdir="TB") + sg.attr(ranksep="0.02") + sg.attr(label="recirc_id {}".format(recirc)) + + invis = "f{}".format(recirc) + sg.node(invis, color="white", len="0", shape="point", width="0", height="0") + + previous = None + for flow in flows: + name = "Flow_{}".format(flow.id) + summary = "Line: {} \n".format(flow.id) + summary += "\n".join( + [ + flow.section("info").string, + ",".join(flow.match.keys()), + "actions: " + ",".join(list(a.keys())[0] for a in flow.actions), + ] + ) + attr = ( + node_styles.get( + next(filter(lambda f: f.evaluate(flow), node_styles), None) + ) + or {} + ) + + sg.node( + name=name, + label=summary, + _attributes=attr, + fontsize="8", + nojustify="true", + URL="#flow_{}".format(flow.id), + ) + + if previous: + sg.edge(previous, name, color="white") + else: + sg.edge(invis, name, color="white", length="0") + previous = name + + next_recirc = next( + (kv.value for kv in flow.actions_kv if kv.key == "recirc"), None + ) + if next_recirc: + cname = "cluster_{}".format(hex(next_recirc)) + g.edge(name, "f{}".format(hex(next_recirc)), lhead=cname) + else: + g.edge(name, "end") + + g.edge("start", "f0x0", lhead="cluster_0x0") + g.node("start", shape="Mdiamond") + g.node("end", shape="Msquare") + + if not html: + print(g.source) + return + + html_obj = "" + html_obj += "

Flow Graph

" + html_obj += "
" + svg = g.pipe(format="svg") + html_obj += svg.decode("utf-8") + html_obj += "
" + + html_obj += get_html_obj(list(itertools.chain(*recirc_flows.values()))) + print(html_obj) + + +class HTMLFlowTree: + def __init__(self, flow=None, opts=None): + self._flow = flow + self._formatter = HTMLFormatter(opts) + self._subflows = list() + + def append(self, flow): + self._subflows.append(flow) + + def render(self, item=0): + html_obj = "
" + if self._flow: + html_obj += """ + + + """.format( + item=item, id=self._flow.id + ) + html_obj += '
'.format( + id=self._flow.id + ) + buf = HTMLBuffer() + self._formatter.format_flow(buf, self._flow, self._style) + html_obj += buf.text + html_obj += "
" + if self._subflows: + html_obj += "
" + html_obj += "" + html_obj += "
" + html_obj += "
" + return html_obj, item + + +def get_html_obj(flow_list, flow_attrs=None): + def append_to_html(parent, flow): + html_flow = HTMLFlowTree(flow) + parent.append(html_flow) + return html_flow + + root = HTMLFlowTree() + process_flow_tree(flow_list, root, 0, append_to_html) + + html_obj = """ + + + + """ + html_obj += """ + + + """ + html_obj += "
" + (html_elem, items) = root.render() + html_obj += html_elem + html_obj += "
" + return html_obj + + +def process_flow_tree(flow_list, parent, recirc_id, callback): + """Process the datapath flows into a tree by "recirc_id" and sorted by "packets" + Args: + flow_list (list[odp.ODPFlow]): original list of flows + parent (Any): current tree node that serves as parent + recirc_id (int): recirc_id to traverse + callback(callable): a callback that must accept the current parent and + a flow and return an object that can potentially serve as parent for + a nested call to callback + + This function is recursive """ - HSV_tuples = [(x / size, 0.7, 0.8) for x in range(size)] - return map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples) + sorted_flows = sorted( + filter(lambda f: f.match.get("recirc_id") == recirc_id, flow_list), + key=lambda x: x.info.get("packets") or 0, + reverse=True, + ) + + for flow in sorted_flows: + next_recirc = next( + (kv.value for kv in flow.actions_kv if kv.key == "recirc"), None + ) + + next_parent = callback(parent, flow) + + if next_recirc: + process_flow_tree(flow_list, next_parent, next_recirc, callback) diff --git a/ovs_dbg/ofparse/format.py b/ovs_dbg/ofparse/format.py new file mode 100644 index 0000000..af53da6 --- /dev/null +++ b/ovs_dbg/ofparse/format.py @@ -0,0 +1,304 @@ +""" This module defines generic formatting classes +""" + + +class FlowStyle: + """ + A FlowStyle determines the KVStyle to use for each key value in a flow + + Styles are internally represented by a dictionary. + In order to determine the style for a "key", the following items in the + dictionary are fetched: + - key.highlighted.{key} (if key is found in hightlighted) + - key.highlighted (if key is found in hightlighted) + - key.{key} + - key + - default + + In order to determine the style for a "value", the following items in the + dictionary are fetched: + - value.highlighted.{key} (if key is found in hightlighted) + - value.highlighted.type{value.__class__.__name__} + - value.highlighted + (if key is found in hightlighted) + - value.{key} + - value.type.{value.__class__.__name__} + - value + - default + + Additionally, the following style items can be defined: + - delim: for delimiters + - delim.highlighted: for delimiters of highlighted key-values + """ + + def __init__(self, initial=None): + self._styles = initial if initial is not None else dict() + + def set_flag_style(self, kvstyle): + self._styles["flag"] = kvstyle + + def set_delim_style(self, kvstyle, highlighted=False): + if highlighted: + self._styles["delim.highlighted"] = kvstyle + else: + self._styles["delim"] = kvstyle + + def set_default_key_style(self, kvstyle): + self._styles["key"] = kvstyle + + def set_default_value_style(self, kvstyle): + self._styles["value"] = kvstyle + + def set_key_style(self, key, kvstyle, highlighted=False): + if highlighted: + self._styles["key.highlighted.{}".format(key)] = kvstyle + else: + self._styles["key.{}".format(key)] = kvstyle + + def set_value_style(self, key, kvstyle, highlighted=None): + if highlighted: + self._styles["value.highlighted.{}".format(key)] = kvstyle + else: + self._styles["value.{}".format(key)] = kvstyle + + def set_value_type_style(self, name, kvstyle, highlighted=None): + if highlighted: + self._styles["value.highlighted.type.{}".format(name)] = kvstyle + else: + self._styles["value.type.{}".format(name)] = kvstyle + + def get(self, key): + return self._styles.get(key) + + def get_delim_style(self, highlighted=False): + delim_style_lookup = ["delim.highlighted"] if highlighted else [] + delim_style_lookup.extend(["delim", "default"]) + return next( + (self._styles.get(s) for s in delim_style_lookup if self._styles.get(s)), + None, + ) + + def get_flag_style(self): + return self._styles.get("flag") or self._styles.get("default") + + def get_key_style(self, kv, highlighted=False): + key = kv.meta.kstring + + key_style_lookup = ( + ["key.highlighted.%s" % key, "key.highlighted"] if highlighted else [] + ) + key_style_lookup.extend(["key.%s" % key, "key", "default"]) + + style = next( + (self._styles.get(s) for s in key_style_lookup if self._styles.get(s)), + None, + ) + if callable(style): + return style(kv.meta.kstring) + return style + + def get_value_style(self, kv, highlighted=False): + key = kv.meta.kstring + value_type = kv.value.__class__.__name__ + value_style_lookup = ( + [ + "value.highlighted.%s" % key, + "value.highlighted.type.%s" % value_type, + "value.highlighted", + ] + if highlighted + else [] + ) + value_style_lookup.extend( + [ + "value.%s" % key, + "value.type.%s" % value_type, + "value", + "default", + ] + ) + + style = next( + (self._styles.get(s) for s in value_style_lookup if self._styles.get(s)), + None, + ) + if callable(style): + return style(kv.meta.vstring) + return style + + +class FlowFormatter: + """FlowFormatter is a base class for Flow Formatters""" + + def __init__(self): + self._highlighted = list() + + def highlight(self, keys): + """Set the highlighted keys + Args: + keys (list[str]): list of keys to highlight + """ + self._highlighted = keys + + def _style_from_opts(self, opts, opts_key, style_constructor): + """Create style object from options + + Args: + opts (dict): Options dictionary + opts_key (str): The options style key to extract (e.g: console or html) + style_constructor(callable): A callable that creates a derived style object + """ + if not opts or not opts.get("style"): + return None + + section_name = ".".join(["styles", opts.get("style")]) + if section_name not in opts.get("config").sections(): + raise Exception("Style not present in config file") + + config = opts.get("config")[section_name] + style = {} + for key in config: + (_, console, style_full_key) = key.partition(opts_key + ".") + if not console: + continue + + (style_key, _, prop) = style_full_key.rpartition(".") + if not prop or not style_key: + raise Exception("malformed style config: {}".format(key)) + + if not style.get(style_key): + style[style_key] = {} + style[style_key][prop] = config[key] + + return FlowStyle({k: style_constructor(**v) for k, v in style.items()}) + + def format_flow(self, buf, flow, style_obj=None): + """ + Formats the flow into the provided buffer + + Args: + buf (FlowBuffer): the flow buffer to append to + flow (ovs_dbg.OFPFlow): the flow to format + style_obj (FlowStyle): Optional; style to use + """ + last_printed_pos = 0 + style_obj = style_obj or FlowStyle() + + for section in sorted(flow.sections, key=lambda x: x.pos): + buf.append_extra( + flow.orig[last_printed_pos : section.pos], + style=style_obj.get("default"), + ) + self.format_kv_list(buf, section.data, section.string, style_obj) + last_printed_pos = section.pos + len(section.string) + + def format_kv_list(self, buf, kv_list, full_str, style_obj): + """ + Format a KeyValue List + + Args: + buf (FlowBuffer): a FlowBuffer to append formatted KeyValues to + kv_list (list[KeyValue]: the KeyValue list to format + full_str (str): the full string containing all k-v + style_obj (FlowStyle): a FlowStyle object to use + """ + for i in range(len(kv_list)): + kv = kv_list[i] + written = self.format_kv(buf, kv, style_obj=style_obj) + + end = kv_list[i + 1].meta.kpos if i < (len(kv_list) - 1) else len(full_str) + + buf.append_extra( + full_str[(kv.meta.kpos + written) : end].rstrip("\n\r"), + style=style_obj.get("default"), + ) + + def format_kv(self, buf, kv, style_obj): + """Format a KeyValue + + A formatted keyvalue has the following parts: + {key}{delim}{value}[{delim}] + + Args: + buf (FlowBuffer): buffer to append the KeyValue to + kv (KeyValue): The KeyValue to print + style_obj (FlowStyle): The style object to use + + Returns the number of printed characters + """ + ret = 0 + key = kv.meta.kstring + highlighted = key in self._highlighted + + key_style = style_obj.get_key_style(kv, highlighted) + buf.append_key(kv, key_style) # format value + ret += len(key) + + if not kv.meta.vstring: + return ret + + if kv.meta.delim not in ("\n", "\t", "\r", ""): + buf.append_delim(kv, style_obj.get_delim_style(highlighted)) + ret += len(kv.meta.delim) + + value_style = style_obj.get_value_style(kv, highlighted) + + buf.append_value(kv, value_style) # format value + ret += len(kv.meta.vstring) + + if kv.meta.end_delim: + buf.append_end_delim(kv, style_obj.get_delim_style(highlighted)) + ret += len(kv.meta.end_delim) + + return ret + + +class FlowBuffer: + """A FlowBuffer is a base class for format buffers. + Childs must implement the following methods: + append_key(self, kv, style) + append_value(self, kv, style) + append_delim(self, delim, style) + append_end_delim(self, delim, style) + append_extra(self, extra, style) + """ + + def append_key(self, kv, style): + """Append a key + Args: + kv (KeyValue): the KeyValue instance to append + style (Any): the style to use + """ + raise Exception("Not implemented") + + def append_delim(self, kv, style): + """Append a delimiter + Args: + kv (KeyValue): the KeyValue instance to append + style (Any): the style to use + """ + raise Exception("Not implemented") + + def append_end_delim(self, kv, style): + """Append an end delimiter + Args: + kv (KeyValue): the KeyValue instance to append + style (Any): the style to use + """ + raise Exception("Not implemented") + + def append_value(self, kv, style): + """Append a value + Args: + kv (KeyValue): the KeyValue instance to append + style (Any): the style to use + """ + raise Exception("Not implemented") + + def append_extra(self, extra, style): + """Append extra string + Args: + kv (KeyValue): the KeyValue instance to append + style (Any): the style to use + """ + raise Exception("Not implemented") diff --git a/ovs_dbg/ofparse/html.py b/ovs_dbg/ofparse/html.py new file mode 100644 index 0000000..aac4591 --- /dev/null +++ b/ovs_dbg/ofparse/html.py @@ -0,0 +1,114 @@ +from ovs_dbg.ofparse.format import FlowFormatter, FlowBuffer, FlowStyle + + +class HTMLStyle(FlowStyle): + """HTMLStyle defines a style for html-formatted flows + + Args: + color(str): Optional; a string representing the CSS color to use + anchor_gen(callable): Optional; a callable to be used to generate the + href + """ + + def __init__(self, color=None, anchor_gen=None): + self.color = color + self.anchor_gen = anchor_gen + + +class HTMLBuffer(FlowBuffer): + """HTMLBuffer implementes FlowBuffer to provide html-based flow formatting + + Each flow gets formatted as: +
...
+ """ + + def __init__(self): + self._text = "" + + @property + def text(self): + return self._text + + def _append(self, string, color, href): + """Append a key a string""" + style = ' style="color:{}"'.format(color) if color else "" + self._text += "".format(style) + if href: + self._text += "".format(href) + self._text += string + if href: + self._text += "" + self._text += "" + + def append_key(self, kv, style): + """Append a key + Args: + kv (KeyValue): the KeyValue instance to append + style (HTMLStyle): the style to use + """ + href = style.anchor_gen(kv) if (style and style.anchor_gen) else "" + return self._append(kv.meta.kstring, style.color if style else "", href) + + def append_delim(self, kv, style): + """Append a delimiter + Args: + kv (KeyValue): the KeyValue instance to append + style (HTMLStyle): the style to use + """ + href = style.anchor_gen(kv) if (style and style.anchor_gen) else "" + return self._append(kv.meta.delim, style.color if style else "", href) + + def append_end_delim(self, kv, style): + """Append an end delimiter + Args: + kv (KeyValue): the KeyValue instance to append + style (HTMLStyle): the style to use + """ + href = style.anchor_gen(kv) if (style and style.anchor_gen) else "" + return self._append(kv.meta.end_delim, style.color if style else "", href) + + def append_value(self, kv, style): + """Append a value + Args: + kv (KeyValue): the KeyValue instance to append + style (HTMLStyle): the style to use + """ + href = style.anchor_gen(kv) if (style and style.anchor_gen) else "" + return self._append(kv.meta.vstring, style.color if style else "", href) + + def append_extra(self, extra, style): + """Append extra string + Args: + kv (KeyValue): the KeyValue instance to append + style (HTMLStyle): the style to use + """ + return self._append(extra, style.color if style else "", "") + + +class HTMLFormatter(FlowFormatter): + """ + Formts a flow in HTML Format + """ + + default_style_obj = FlowStyle( + { + "value.resubmit": HTMLStyle( + anchor_gen=lambda x: "#table_{}".format(x.value["table"]) + ), + "default": HTMLStyle(), + } + ) + + def __init__(self, opts=None): + super(HTMLFormatter, self).__init__() + self.style = self._style_from_opts(opts, "html", HTMLStyle) or FlowStyle() + self.style.set_value_style( + "resubmit", + HTMLStyle( + self.style.get("value.resubmit"), + anchor_gen=lambda x: "#table_{}".format(x.value["table"]), + ), + ) + + def format_flow(self, buf, flow, style=None): + return super(HTMLFormatter, self).format_flow(buf, flow, self.style) diff --git a/ovs_dbg/ofparse/main.py b/ovs_dbg/ofparse/main.py index 75a3ae2..2b71a7a 100644 --- a/ovs_dbg/ofparse/main.py +++ b/ovs_dbg/ofparse/main.py @@ -1,7 +1,12 @@ import click import sys +import configparser from ovs_dbg.filter import OFFilter +from pkg_resources import resource_filename + +_default_config_file = "ofparse.conf" +_default_config_path = resource_filename(__name__, _default_config_file) class Options(dict): @@ -13,6 +18,18 @@ class Options(dict): @click.group( subcommand_metavar="TYPE", context_settings=dict(help_option_names=["-h", "--help"]) ) +@click.option( + "-c", + "--config", + help="Use non-default config file", + type=click.Path(), +) +@click.option( + "--style", + help="Select style (defined in config file)", + default=None, + show_default=True, +) @click.option( "-i", "-input", @@ -24,15 +41,8 @@ class Options(dict): @click.option( "-p", "--paged", - help="Page the result (uses $PAGER). If styling is not disabled you might " - 'need to enable colors on your $PAGER, eg: export PAGER="less -r".', - is_flag=True, - default=False, - show_default=True, -) -@click.option( - "--no-style", - help="Do not styles (colors)", + help="Page the result (uses $PAGER). If colors are not disabled you might " + 'need to enable colors on your PAGER, eg: export PAGER="less -r".', is_flag=True, default=False, show_default=True, @@ -46,7 +56,7 @@ class Options(dict): show_default=False, ) @click.pass_context -def maincli(ctx, filename, paged, no_style, filter): +def maincli(ctx, config, style, filename, paged, filter): """ OpenFlow Parse utility. @@ -57,13 +67,19 @@ def maincli(ctx, filename, paged, no_style, filter): ctx.obj = Options() ctx.obj["filename"] = filename or "" ctx.obj["paged"] = paged - ctx.obj["no_style"] = no_style if filter: try: ctx.obj["filter"] = OFFilter(filter) except Exception as e: raise click.BadParameter("Wrong filter syntax: {}".format(e)) + config_file = config or _default_config_path + parser = configparser.ConfigParser() + parser.read(config_file) + + ctx.obj["config"] = parser + ctx.obj["style"] = style + @maincli.command(hidden=True) @click.pass_context diff --git a/ovs_dbg/ofparse/ofp.py b/ovs_dbg/ofparse/ofp.py index abdb7dc..0cc1dff 100644 --- a/ovs_dbg/ofparse/ofp.py +++ b/ovs_dbg/ofparse/ofp.py @@ -7,10 +7,16 @@ from rich.style import Style from rich.color import Color +from ovs_dbg.ofp import OFPFlow from ovs_dbg.ofparse.main import maincli from ovs_dbg.ofparse.process import process_flows, tojson, pprint -from .console import OFConsole, print_context -from ovs_dbg.ofp import OFPFlow +from ovs_dbg.ofparse.console import ( + ConsoleBuffer, + ConsoleFormatter, + hash_pallete, + print_context, +) +from ovs_dbg.ofparse.html import HTMLBuffer, HTMLFormatter @maincli.group(subcommand_metavar="FORMAT") @@ -24,14 +30,14 @@ def openflow(opts): @click.pass_obj def json(opts): """Print the flows in JSON format""" - return tojson(flow_factory=OFPFlow.from_string, opts=opts) + return tojson(flow_factory=create_ofp_flow, opts=opts) @openflow.command() @click.pass_obj def pretty(opts): """Print the flows with some style""" - return pprint(flow_factory=OFPFlow.from_string, opts=opts) + return pprint(flow_factory=create_ofp_flow, opts=opts) @openflow.command() @@ -101,10 +107,8 @@ def callback(flow): tables[table][lflow].append(flow) - console = OFConsole() - process_flows( - flow_factory=OFPFlow.from_string, + flow_factory=create_ofp_flow, callback=callback, filename=opts.get("filename"), filter=opts.get("filter"), @@ -112,13 +116,14 @@ def callback(flow): # Try to make it easy to spot same cookies by printing them in different # colors - cookie_styles = [ - Style(color=Color.from_rgb(r * 255, g * 255, b * 255)) - for r, g, b in create_color_pallete(200) - ] + cookie_style_gen = hash_pallete( + hue=[x / 10 for x in range(0, 10)], + saturation=[0.5], + value=[0.5 + x / 10 * (0.85 - 0.5) for x in range(0, 10)], + ) tree = Tree("Ofproto Flows (logical)") - console = Console(color_system="256") + console = Console(color_system=None if opts["style"] is None else "256") for table_num in sorted(tables.keys()): table = tables[table_num] @@ -131,32 +136,66 @@ def callback(flow): ): flows = table[lflow] - text = Text() + buf = ConsoleBuffer(Text()) - text.append( + buf.append_extra( "cookie={} ".format(hex(lflow.cookie)).ljust(18), - style=cookie_styles[(lflow.cookie * 0x27D4EB2D) % len(cookie_styles)], + style=cookie_style_gen(str(lflow.cookie)), ) - text.append("priority={} ".format(lflow.priority), style="steel_blue") - text.append(",".join(lflow.match_keys), style="steel_blue") - text.append(" ---> ", style="bold magenta") - text.append(",".join(lflow.action_keys), style="steel_blue") - text.append(" ( x {} )".format(len(flows)), style="dark_olive_green3") - lflow_tree = table_tree.add(text) + buf.append_extra("priority={} ".format(lflow.priority), style="steel_blue") + buf.append_extra(",".join(lflow.match_keys), style="steel_blue") + buf.append_extra(" ---> ", style="bold magenta") + buf.append_extra(",".join(lflow.action_keys), style="steel_blue") + buf.append_extra(" ( x {} )".format(len(flows)), style="dark_olive_green3") + lflow_tree = table_tree.add(buf.text) if show_flows: for flow in flows: - text = Text() - OFConsole(console).format_flow(flow, text=text) - lflow_tree.add(text) + buf = ConsoleBuffer(Text()) + ConsoleFormatter(console, opts).format_flow(buf, flow) + lflow_tree.add(buf.text) - with print_context(console, opts["paged"], not opts["no_style"]): + with print_context(console, opts): console.print(tree) -def create_color_pallete(size): - """Create a color pallete of size colors by modifying the Hue in the HSV - color space - """ - HSV_tuples = [(x / size, 0.5, 0.5) for x in range(size)] - return map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples) +@openflow.command() +@click.pass_obj +def html(opts): + """Print the flows in an HTML list""" + tables = dict() + + def callback(flow): + """Parse the flows and sort them by table""" + table = flow.info.get("table") or 0 + if not tables.get(table): + tables[table] = list() + tables[table].append(flow) + + process_flows( + flow_factory=create_ofp_flow, + callback=callback, + filename=opts.get("filename"), + filter=opts.get("filter"), + ) + + html_obj = "
" + for table, flows in tables.items(): + html_obj += "

Table {table}

".format(table=table) + html_obj += "" + html_obj += "
" + print(html_obj) + + +def create_ofp_flow(string, idx): + """Create a OFPFlow""" + if " reply " in string: + return None + return OFPFlow.from_string(string, idx) diff --git a/ovs_dbg/ofparse/process.py b/ovs_dbg/ofparse/process.py index fa45018..773de4d 100644 --- a/ovs_dbg/ofparse/process.py +++ b/ovs_dbg/ofparse/process.py @@ -6,7 +6,7 @@ from ovs_dbg.ofp import OFPFlow from ovs_dbg.decoders import FlowEncoder -from ovs_dbg.ofparse.console import OFConsole, print_context +from ovs_dbg.ofparse.console import ConsoleFormatter, print_context def process_flows(flow_factory, callback, filename="", filter=None): @@ -18,11 +18,13 @@ def process_flows(flow_factory, callback, filename="", filter=None): filename (str): Optional; filename to read frows from filter (OFFilter): Optional; filter to use to filter flows """ + idx = 0 if filename: with open(filename) as f: for line in f: - flow = flow_factory(line) - if filter and not filter.evaluate(flow): + flow = flow_factory(line, idx) + idx += 1 + if not flow or (filter and not filter.evaluate(flow)): continue callback(flow) else: @@ -30,8 +32,9 @@ def process_flows(flow_factory, callback, filename="", filter=None): for line in data.split("\n"): line = line.strip() if line: - flow = flow_factory(line) - if filter and not filter.evaluate(flow): + flow = flow_factory(line, idx) + idx += 1 + if not flow or (filter and not filter.evaluate(flow)): continue callback(flow) @@ -59,25 +62,24 @@ def callback(flow): if opts["paged"]: console = rich.Console() - with print_context(console, opts["paged"], not opts["no_style"]): + with print_context(console, opts): console.print(flow_json) else: print(flow_json) -def pprint(flow_factory, opts, style=None): +def pprint(flow_factory, opts): """ Pretty print the flows Args: flow_factory (Callable): Function to call to create the flows opts (dict): Options - style (dict): Optional, Style dictionary """ - console = OFConsole(no_color=opts["no_style"]) + console = ConsoleFormatter(opts) def callback(flow): - console.print_flow(flow, style=style) + console.print_flow(flow) - with print_context(console.console, opts["paged"], not opts["no_style"]): + with print_context(console.console, opts): process_flows(flow_factory, callback, opts.get("filename"), opts.get("filter")) diff --git a/setup.py b/setup.py index c8dd63d..db1160f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ with open("README.md") as readme_file: readme = readme_file.read() -requirements = ["click", "rich", "pyparsing", "netaddr"] +requirements = ["click", "rich", "pyparsing", "netaddr", "graphviz"] setup_requirements = [ "pytest-runner",