Merge remote-tracking branch 'origin/develop'

docs/release-notes.md

@@ -1,5 +1,15 @@
 # Release Notes
 
+## 0.19.1 (Sep 6, 2022)
+
+This patch release for the 0.19.0 release includes the following important fixes:
+
+* Fixes bug in path whereby an address such as 10.1.1.110 was incorrectly matched with 10.1.1.1 causing path problems
+* Fixes inconsistent use of "!" in filtering by device version string
+* Fixes intermittent crashes in device show (or the status page in the GUI) when using columns=*
+* Fixes incorrect hostname setting on NXOS devices in some conditions when the hostname is a hostname that includes the domain name
+* Add missing keyword query-str to path commands
+
 ## 0.19.0 (Aug 22, 2022)
 
 The 19th release of SuzieQ contains bug fixes and improvements to key functionalities such as the REST API and endpoint tracker. Here's a detailed list of key features of this release:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "suzieq"
-version = "0.19.0"
+version = "0.19.1"
 description = "A framework and application for network observability"
 readme = 'README.md'
 repository = 'https://github.com/netenglabs/suzieq'

suzieq/cli/sqcmds/PathCmd.py

@@ -25,6 +25,7 @@ def __init__(
         src: str = "",
         dest: str = "",
         vrf: str = "",
+        query_str: str = ' ',
     ) -> None:
         super().__init__(
             engine=engine,
@@ -34,6 +35,7 @@ def __init__(
             view=view,
             namespace=namespace,
             columns=columns,
+            query_str=query_str,
             format=format,
             sqobj=PathObj
         )

suzieq/engines/pandas/device.py

@@ -84,6 +84,7 @@ def get(self, **kwargs):
                 (df['status_y'] != 0) & (df['status_y'] != 200) &
                 (df['status'] == "N/A"),
                 'neverpoll', df['status'])
+
             if 'version' in df.columns:
                 df['version'] = np.where(df.status_y == 418, 'unsupported',
                                          df.version)
@@ -100,13 +101,13 @@ def get(self, **kwargs):
                 df.address = np.where(df['address'] == 'N/A', df['hostname'],
                                       df['address'])
 
-            if 'uptime' in columns or columns == ['*']:
-                uptime_cols = (df['timestamp'] -
-                               humanize_timestamp(df['bootupTimestamp']*1000,
-                               self.cfg.get('analyzer', {}).get('timezone',
-                                                                None)))
-                uptime_cols = pd.to_timedelta(uptime_cols, unit='s')
-                df.insert(len(df.columns)-1, 'uptime', uptime_cols)
+        if 'uptime' in columns or columns == ['*']:
+            uptime_cols = (df['timestamp'] -
+                           humanize_timestamp(df['bootupTimestamp']*1000,
+                           self.cfg.get('analyzer', {}).get('timezone',
+                                                            None)))
+            uptime_cols = pd.to_timedelta(uptime_cols, unit='s')
+            df.insert(len(df.columns)-1, 'uptime', uptime_cols)
 
         if df.empty:
             return df[fields]
@@ -116,9 +117,11 @@ def get(self, **kwargs):
             df = df.loc[df.status.isin(status)]
         if os_version:
             opdict = {'>': operator.gt, '<': operator.lt, '>=': operator.ge,
-                      '<=': operator.le, '=': operator.eq, '!=': operator.ne}
+                      '<=': operator.le, '=': operator.eq, '!': operator.ne}
             op = operator.eq
             for osv in os_version:
+                # Introduced in 0.19.1, we do this for backwards compatibility
+                osv = osv.replace('!=', '!')
                 for elem, val in opdict.items():
                     if osv.startswith(elem):
                         osv = osv.replace(elem, '')

suzieq/engines/pandas/path.py

@@ -119,10 +119,10 @@ def _init_dfs(self, ns, source, dest):
 
         if ':' in source:
             self._src_df = self._if_df[self._if_df.ip6AddressList.astype(str)
-                                       .str.contains(source + "/")]
+                                       .str.startswith(source + "/")]
         else:
             self._src_df = self._if_df[self._if_df.ipAddressList.astype(str)
-                                       .str.contains(source + "/")]
+                                       .str.startswith(source + "/")]
 
         if self._src_df.empty:
             # TODO: No host with this src addr. Is addr a local ARP entry?
@@ -147,10 +147,10 @@ def _init_dfs(self, ns, source, dest):
 
         if ':' in dest:
             self._dest_df = self._if_df[self._if_df.ip6AddressList.astype(str)
-                                        .str.contains(dest + "/")]
+                                        .str.startswith(dest + "/")]
         else:
             self._dest_df = self._if_df[self._if_df.ipAddressList.astype(str)
-                                        .str.contains(dest + "/")]
+                                        .str.startswith(dest + "/")]
 
         if self._dest_df.empty:
             # Check if addr is in any switch's local ARP
@@ -752,6 +752,7 @@ def get(self, **kwargs) -> pd.DataFrame:
         src = kwargs.get("src", None)
         dest = kwargs.get("dest", None)
         dvrf = kwargs.get("vrf", "")
+        query_str = kwargs.pop('query_str', "")
 
         if not src or not dest:
             raise AttributeError("Must specify trace source and dest")
@@ -929,10 +930,10 @@ def get(self, **kwargs) -> pd.DataFrame:
                     if not nhdf.empty:
                         if srcvers == 4:
                             nhdf = nhdf.query(
-                                f'ipAddressList.str.contains("{ndst}")')
+                                f'ipAddressList.str.startswith("{ndst}/")')
                         else:
                             nhdf = nhdf.query(
-                                f'ip6AddressList.str.contains("{ndst}")')
+                                f'ip6AddressList.str.startswith("{ndst}/")')
                     if not nhdf.empty:
                         ifmac = nhdf.macaddr.unique().tolist()
                         if (not macaddr) or (macaddr in ifmac):
@@ -1120,7 +1121,9 @@ def get(self, **kwargs) -> pd.DataFrame:
             # This occurs when a path traversal terminates due to an error such
             # as loop detected
             final_paths = paths
-        return self._path_cons_result(final_paths)
+        return self._handle_user_query_str(
+            self._path_cons_result(final_paths), query_str)\
+            .reset_index(drop=True)
 
     def _path_cons_result(self, paths):
         df_plist = []

suzieq/engines/pandas/topology.py

@@ -23,6 +23,13 @@
 class TopologyObj(SqPandasEngine):
     '''Backend class to operate on virtual table, topology, with pandas'''
 
+    def __init__(self, baseobj):
+        super().__init__(baseobj)
+        self.lsdb = pd.DataFrame()
+        self._a_df = pd.DataFrame()
+        self._ip_table = pd.DataFrame()
+        self.nses = []
+
     @staticmethod
     def table_name():
         '''Table name'''
@@ -86,9 +93,6 @@ class Services:
         self._init_dfs(self._namespaces)
         if self._if_df.empty:
             return pd.DataFrame({'error': ['No interfaces data found']})
-        self.lsdb = pd.DataFrame()
-        self._a_df = pd.DataFrame()
-        self._ip_table = pd.DataFrame()
 
         fields = self.schema.get_display_fields(columns)
 

suzieq/poller/worker/nodes/node.py

@@ -409,7 +409,11 @@ async def _parse_device_type_hostname(self, output, _) -> None:
                 f'Detected {devtype} for {self.address}:{self.port},'
                 f' {hostname}')
             self._set_devtype(devtype, version_str)
-            self._set_hostname(hostname)
+            # We don't set the hostname here because the real hostname
+            # is retrieved via a different command on some platforms
+            # and can contain the FQDN. The hostname stored with a
+            # record needs to be one that is also used in LLDP so that we
+            # can find interface peers
             self.current_exception = None
 
     async def _detect_node_type(self):

suzieq/poller/worker/services/service.py

@@ -628,6 +628,7 @@ async def commit_data(self, result: Dict, namespace: str, hostname: str):
                 hostname=[hostname],
                 namespace=[namespace]).query('active')
             prev_res = df.to_dict('records')
+            self.previous_results[key] = prev_res
 
         if result or prev_res:
             adds, dels = self.get_diff(prev_res, result)

suzieq/version.py

@@ -2,7 +2,7 @@
 """Store the Suzieq version string."""
 
 
-SUZIEQ_VERSION = "0.19.0"
+SUZIEQ_VERSION = "0.19.1"
 
 if __name__ == '__main__':
     print(SUZIEQ_VERSION)

tests/integration/sqcmds/common-samples/common-errors.yml

@@ -0,0 +1,6 @@
+description: Testing some common errors independent of NOS
+tests:
+- command: topology summarize --namespace=whatever --format=json
+  data-directory: tests/data/parquet/
+  marks: topology summarize
+  output: '{}'

tests/integration/sqcmds/common-samples/not.yml

@@ -102769,3 +102769,24 @@ tests:
     ["ae1", "ae2", "ge-0/0/0"], "vlan": 10, "timestamp": 1631009089744}, {"namespace":
     "vmx", "hostname": "CRP-DIS-SW01", "vlanName": "vl20", "state": "active", "interfaces":
     ["ae1", "ae2", "ge-0/0/0"], "vlan": 20, "timestamp": 1631009089744}]'
+- command: device show --version="!9.3(4)" --namespace=nxos --format=json
+  data-directory: tests/data/parquet/
+  marks: device show negate
+  output: '[{"namespace": "nxos", "hostname": "server101", "model": "vm", "version":
+    "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive",
+    "address": "10.255.2.204", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256203},
+    {"namespace": "nxos", "hostname": "server102", "model": "vm", "version": "18.04.3
+    LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive", "address":
+    "10.255.2.39", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256290}, {"namespace":
+    "nxos", "hostname": "server301", "model": "vm", "version": "18.04.3 LTS", "vendor":
+    "Ubuntu", "architecture": "x86-64", "status": "alive", "address": "10.255.2.140",
+    "bootupTimestamp": 1619182381.0, "timestamp": 1619275256319}, {"namespace": "nxos",
+    "hostname": "server302", "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu",
+    "architecture": "x86-64", "status": "alive", "address": "10.255.2.114", "bootupTimestamp":
+    1619182381.0, "timestamp": 1619275256394}, {"namespace": "nxos", "hostname": "firewall01",
+    "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64",
+    "status": "alive", "address": "10.255.2.249", "bootupTimestamp": 1619013132.0,
+    "timestamp": 1619275256497}, {"namespace": "nxos", "hostname": "dcedge01", "model":
+    "vqfx-10000", "version": "19.4R1.10", "vendor": "Juniper", "architecture": "",
+    "status": "alive", "address": "10.255.2.250", "bootupTimestamp": 1619013129.329,
+    "timestamp": 1619275258329}]'

tests/integration/sqcmds/cumulus-samples/badquery.yml

@@ -75,8 +75,9 @@ tests:
 - command: path show --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp'
     --query-str='vendor == "Cisco" and mtu==1500' --format=json
   data-directory: tests/data/parquet/
+  error:
+    error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]'
   marks: path show badquery cumulus
-  output: '[]'
 - command: topology show --query-str='vendor == "Cisco" and mtu==1500' --format=json
     --namespace='ospf-single dual-evpn ospf-ibgp'
   data-directory: tests/data/parquet/
@@ -158,8 +159,9 @@ tests:
 - command: path summarize --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp'
     --query-str='vendor == "Cisco" and mtu==1500' --format=json
   data-directory: tests/data/parquet/
+  error:
+    error: '{"error": {"0": "ERROR: UserQueryError: name ''vendor'' is not defined"}}'
   marks: path summarize badquery cumulus
-  output: '[]'
 - command: topology summarize --query-str='vendor == "Cisco" and mtu==1500' --format=json
     --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname'
   data-directory: tests/data/parquet/
@@ -241,8 +243,9 @@ tests:
 - command: path unique --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp'
     --query-str='vendor == "Cisco" and mtu==1500' --format=json --columns='hostname'
   data-directory: tests/data/parquet/
+  error:
+    error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]'
   marks: path unique badquery cumulus
-  output: '[]'
 - command: topology unique --query-str='vendor == "Cisco" and mtu==1500' --format=json
     --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname'
   data-directory: tests/data/parquet/