Sitermlib (#298)

* Fix SiteRMLibs Package name

* Use new ansible modules

src/python/SiteFE/LookUpService/modules/rdfhelper.py

@@ -161,17 +161,18 @@ def _addPort(self, **kwargs):
         self._addNode(**kwargs)
         if not kwargs['hostname'] or not kwargs['portName']:
             return ""
+        newuri = f":{kwargs['hostname']}:{self.switch.getSystemValidPortName(kwargs['portName'])}"
         self.newGraph.add((self.genUriRef('site', f":{kwargs['hostname']}"),
                            self.genUriRef('nml', 'hasBidirectionalPort'),
-                           self.genUriRef('site', f":{kwargs['hostname']}:{kwargs['portName']}")))
-        self.newGraph.add((self.genUriRef('site', f":{kwargs['hostname']}:{kwargs['portName']}"),
+                           self.genUriRef('site', newuri)))
+        self.newGraph.add((self.genUriRef('site', newuri),
                            self.genUriRef('rdf', 'type'),
                            self.genUriRef('nml', 'BidirectionalPort')))
         if 'parent' in kwargs and kwargs['parent']:
             self.newGraph.add((self.genUriRef('site', f":{kwargs['hostname']}:{kwargs['parent']}"),
                                self.genUriRef('nml', 'hasBidirectionalPort'),
-                               self.genUriRef('site', f":{kwargs['hostname']}:{kwargs['portName']}")))
-        return f":{kwargs['hostname']}:{kwargs['portName']}"
+                               self.genUriRef('site', newuri)))
+        return newuri
 
     def _addSwitchingService(self, **kwargs):
         """Add Switching Service to Model"""
@@ -422,7 +423,8 @@ def _addVlanPort(self, **kwargs):
         """Add Vlan Port to Model"""
         if not kwargs['vlan'] and not kwargs['vtype']:
             return ""
-        vlanuri = f":{kwargs['hostname']}:{kwargs['portName']}:{kwargs['vtype']}+{kwargs['vlan']}"
+        uri = self._addPort(**kwargs)
+        vlanuri = f"{uri}:{kwargs['vtype']}+{kwargs['vlan']}"
         if not kwargs['portName'].startswith('Vlan_'):
             uri = self._addPort(**kwargs)
             self.newGraph.add((self.genUriRef('site', uri),

src/python/SiteFE/LookUpService/modules/switchinfo.py

@@ -183,7 +183,7 @@ def _addSwitchPortInfo(self, key, switchInfo):
             except (NoOptionError, NoSectionError):
                 rst = False
             for portName, portSwitch in list(switchDict.items()):
-                newuri = f":{switchName}:{portName}"
+                newuri = f":{switchName}:{self.switch.getSystemValidPortName(portName)}"
                 self._addVswPort(hostname=switchName, portName=portName, vsw=vsw)
                 self.addSwitchIntfInfo(switchName, portName, portSwitch, newuri)
                 if rst:
@@ -239,11 +239,10 @@ def getSwitchSiteRMName(allMacs, macLookUp):
                 if macLookUp in hMacs:
                     return hName
             return None
-
         for lldpHost, lldpDict in switchInfo['lldp'].items():
             for lldpIntf, intfDict in lldpDict.items():
                 if 'remote_port_id' not in intfDict:
-                    self.logger.debug(f'Remote port id not available from lldp info. lldp enabled? Full port info {intfDict}')
+                    self.logger.debug(f'Remote port id not available from lldp info. lldp enabled? Full port info {lldpHost} {lldpIntf} {intfDict}')
                     continue
                 macName = getSwitchSiteRMName(switchInfo['nametomac'], intfDict['remote_chassis_id'])
                 if not macName:

src/python/SiteRMLibs/Backends/Ansible.py

@@ -25,9 +25,6 @@ def __init__(self, config, sitename):
         self.config = config
         self.sitename = sitename
         self.logger = getLoggingObject(config=self.config, service='SwitchBackends')
-        # cmd counter is used only for command with items (e.g. sonic, p4)
-        # the one switches which do not have ansible modules.
-        self.cmdCounter = 0
         self.ansibleErrs = {}
 
     @staticmethod
@@ -101,84 +98,6 @@ def _applyNewConfig(self, hosts=None):
         self.__getAnsErrors(ansOut)
         return ansOut
 
-    # 0 - command show version, system. Mainly to get mac address, but might use for more info later.
-    # 1 - command to get lldp neighbors details.
-    # 2 - ip route information;
-    # 3 - ipv6 route information
-    # For Dell OS 9 - best is to use show running config;
-    # For Arista EOS - show ip route vrf all | json || show ipv6 route vrf all | json
-    # For Azure Sonic - we use normal ssh and command line. - There is also Dell Sonic Module
-    # but that one depends on sonic-cli - which is broken in latest Azure Image (py2/py3 mainly),
-    # See https://github.com/Azure/SONiC/issues/781
-    def _getMacLLDPRoute(self, hosts=None):
-        """Parser for Mac/LLDP/Route Ansible playbook"""
-        def parserWrapper(num, andsiblestdout):
-            """Parser wrapper to call specific parser function"""
-            cmdList = {0: self.parsers[action].getinfo,
-                       1: self.parsers[action].getlldpneighbors,
-                       2: self.parsers[action].getIPv4Routing,
-                       3: self.parsers[action].getIPv6Routing}
-            tmpOut = {}
-            try:
-                if num not in cmdList:
-                    self.logger.info('UNDEFINED FUNCTION num X. Return empty')
-                else:
-                    tmpOut = cmdList[num](andsiblestdout)
-            except NotImplementedError as ex:
-                self.logger.debug(f"Got Not Implemented Error. {ex}")
-            except (AttributeError, IndexError) as ex:
-                self.logger.debug(f'Got Exception calling switch module for {action} and Num {num}. Error: {ex}')
-            return tmpOut
-
-        keyMapping = {0: 'info', 1: 'lldp', 2: 'ipv4', 3: 'ipv6'}
-        out = {}
-        ansOut = {}
-        try:
-            ansOut = self._executeAnsible('maclldproute.yaml', hosts)
-        except ValueError as ex:
-            raise ConfigException(f"Got Value Error. Ansible configuration exception {ex}") from ex
-        for host, _ in ansOut.stats['ok'].items():
-            hOut = out.setdefault(host, {})
-            for host_events in ansOut.host_events(host):
-                if host_events['event'] not in ['runner_on_ok', 'runner_item_on_ok']:
-                    continue
-                if 'stdout' in host_events['event_data']['res']:
-                    # 0 - command to get mainly mac
-                    action = host_events['event_data']['task_action']
-                    # In case of command, we pass most event_data back to Backend parser
-                    # because it does not group output in a single ansible even
-                    # as ansible network modules.
-                    if action == 'command':
-                        # This means it is not using any special ansible module
-                        # to communicate with switch/router. In this case, we get
-                        # ansible_network_os and use that for loading module
-                        action = f"{self.getAnsNetworkOS(host)}_command"
-                        # And in case action is not set - means it is badly configured
-                        # inside the ansible module by sys/net admin.
-                        # We log this inside te log, and ignore that switch
-                        if action == '_command':
-                            self.logger.info(f'WARNING. ansible_network_os is not defined for {host} host. Ignoring this host')
-                            continue
-                        if action not in self.parsers.keys():
-                            self.logger.info('WARNING. ansible action not defined in site-rm code base. Unsupported switch?')
-                            continue
-                        hOut.setdefault(keyMapping[self.cmdCounter], {})
-                        hOut[keyMapping[self.cmdCounter]] = parserWrapper(self.cmdCounter, host_events['event_data']['res'])
-                        self.cmdCounter += 1
-                    elif action not in self.parsers.keys():
-                        self.logger.info('WARNING. ansible action not defined in site-rm code base. Unsupported switch?')
-                        continue
-                    else:
-                        for val, key in keyMapping.items():
-                            if val < len(host_events.get('event_data', {}).get('res', {}).get('stdout', [])):
-                                hOut.setdefault(key, {})
-                                hOut[key] = parserWrapper(val, host_events['event_data']['res']['stdout'][val])
-            for val, key in keyMapping.items():
-                if key not in hOut:
-                    hOut.setdefault(key, {})
-                    hOut[key] = parserWrapper(val, None)
-        self.__getAnsErrors(ansOut)
-        return out
 
     def _getFacts(self, hosts=None):
         """Get All Facts for all Ansible Hosts"""
@@ -195,35 +114,14 @@ def _getFacts(self, hosts=None):
                 if host_events['event'] != 'runner_on_ok':
                     continue
                 action = host_events['event_data']['task_action']
-                if action == 'command':
-                    # This means it is not using any special ansible module
-                    # to communicate with switch/router. In this case, we get
-                    # ansible_network_os and use that for loading module
-                    action = f"{self.getAnsNetworkOS(host)}_command"
-                    # And in case action is not set - means it is badly configured
-                    # inside the ansible module by sys/net admin.
-                    # We log this inside te log, and ignore that switch
-                    if action == '_command':
-                        self.logger.info(f'WARNING. ansible_network_os is not defined for {host} host. Ignoring this host')
-                        continue
-                ansNetIntf = host_events.setdefault('event_data', {}).setdefault('res', {}).setdefault('ansible_facts', {}).setdefault('ansible_net_interfaces', {})
-                if action in self.parsers.keys():
-                    tmpOut = self.parsers[action].parser(host_events)
-                    for portName, portVals in tmpOut.items():
-                        ansNetIntf.setdefault(portName, {})
-                        ansNetIntf[portName].update(portVals)
-                        host_events['event_data']['res']['ansible_facts']['ansible_net_interfaces'].setdefault(portName, {})
-                        host_events['event_data']['res']['ansible_facts']['ansible_net_interfaces'][portName].update(portVals)
+                if action not in self.parsers.keys():
+                    self.logger.warning(f'Unsupported NOS. There might be issues. Contact dev team')
                 out[host] = host_events
+                ansNetIntf = host_events.setdefault('event_data', {}).setdefault('res', {}).setdefault('ansible_facts',{})
+                print(ansNetIntf.keys())
+                print(out[host].keys())
         self.__getAnsErrors(ansOut)
-        try:
-            maclldproute = self._getMacLLDPRoute(hosts)
-            for host, hitems in maclldproute.items():
-                if host in out:
-                    for key, vals in hitems.items():
-                        out[host]['event_data']['res']['ansible_facts'][f'ansible_command_{key}'] = vals
-        finally:
-            self.cmdCounter = 0
+        # TODO: add lldp, routing info
         return out, self.ansibleErrs
 
     @staticmethod
@@ -265,7 +163,7 @@ def getfactvalues(inData, key):
 
     def nametomac(self, inData, key):
         """Return all mac's associated to that host. Not in use for RAW plugin"""
-        macs = inData.get('event_data', {}).get('res', {}).get('ansible_facts', {}).get('ansible_command_info', {}).get('mac', [])
+        macs = inData.get('event_data', {}).get('res', {}).get('ansible_facts', {}).get('ansible_net_info', {}).get('macs', [])
         if macs and isinstance(macs, str):
             return [macs]
         if macs and isinstance(macs, list):

src/python/SiteRMLibs/Backends/main.py

@@ -84,7 +84,7 @@ def getSystemValidPortName(port):
         # Also - mrml does not expect to get string in nml. so need to replace all
         # Inside the output of dictionary
         # Also - sometimes lldp reports multiple quotes for interface name from ansible out
-        for rpl in [[" ", "_"], ["/", "-"], ['"', ''], ["'", ""]]:
+        for rpl in [[" ", "_"], ["/", "-"], ['"', ''], ["'", ""], [":", "_"]]:
             port = port.replace(rpl[0], rpl[1])
         return port
 
@@ -218,10 +218,11 @@ def _mergeYamlAndSwitch(self, switch):
                 if 'switchport' in portDict.keys() and portDict['switchport']:
                     portDict['desttype'] = 'switch'
         # Add route information and lldp information to output dictionary
-        self.output['info'][switch] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_command_info')
-        self.output['routes'][switch]['ipv4'] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_command_ipv4')
-        self.output['routes'][switch]['ipv6'] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_command_ipv6')
-        self.output['lldp'][switch] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_command_lldp')
+
+        self.output['info'][switch] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_net_info')
+        self.output['routes'][switch]['ipv4'] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_net_ipv4')
+        self.output['routes'][switch]['ipv6'] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_net_ipv6')
+        self.output['lldp'][switch] = self.plugin.getfactvalues(self.switches['output'][switch], 'ansible_net_lldp')
         self.output['nametomac'][switch] = self.plugin.nametomac(self.switches['output'][switch], switch)
 
 

src/python/SiteRMLibs/Backends/parsers/aristaeos.py

@@ -18,127 +18,5 @@ class AristaEOS():
     def __init__(self, **kwargs):
         self.factName = ['arista.eos.eos_facts', 'arista.eos.facts', 'arista.eos.eos_command']
         self.defVlanNaming = 'Vlan%(vlanid)s'
-        self.logger = getLoggingObject(config=kwargs['config'], service='SwitchBackends')
-
-    @staticmethod
-    def getSystemValidPortName(port):
-        """Get Systematic port name. MRML expects it without spaces"""
-        # Spaces from port name are replaced with _
-        # Backslashes are replaced with dash
-        # Also - mrml does not expect to get string in nml. so need to replace all
-        # Inside the output of dictionary
-        # Also - sometimes lldp reports multiple quotes for interface name from ansible out
-        for rpl in [[" ", "_"], ["/", "-"], ['"', ''], ["'", ""]]:
-            port = port.replace(rpl[0], rpl[1])
-        return port
-
-    @staticmethod
-    def _getVlans(inLine):
-        """Get All vlans list assigned to port"""
-        out = []
-        tmpVlans = inLine.split()[-1:][0] # Get the last item from split, e.g. 1127,1779-1799,2803
-        if tmpVlans == 'none':
-            # Arista needs to define none vlans (if none are configured on the link)
-            return out
-        for splPorts in tmpVlans.split(','):
-            splRange = splPorts.split('-')
-            if len(splRange) == 2:
-                for i in range(int(splRange[0]), int(splRange[1]) + 1):
-                    out.append(i)
-            else:
-                out.append(splRange[0])
-        return out
-
-    def parser(self, ansibleOut):
-        """Parse Ansible output and prepare it as other SENSE Services expect it"""
-        # Out must be {'<interface_name>': {'key': 'value'}} OR
-        #             {'<interface_name>': {'key': ['value1', 'value2']}
-        # dict as value are not supported (not found use case yet for this)
-        out = {}
-        interfaceSt = ""
-        for line in ansibleOut['event_data']['res']['ansible_facts']['ansible_net_config'].split('\n'):
-            line = line.strip() # Remove all white spaces
-            if line == "!" and interfaceSt:
-                interfaceSt = "" # This means interface ended!
-            elif line.startswith('interface'):
-                interfaceSt = line[10:]
-            elif interfaceSt:
-                if line.startswith('switchport trunk allowed vlan') or line.startswith('switchport access vlan'):
-                    for vlan in self._getVlans(line):
-                        key = f"Vlan{vlan}"
-                        out.setdefault(key, {})
-                        out[key].setdefault('tagged', [])
-                        out[key]['tagged'].append(self.getSystemValidPortName(interfaceSt))
-                else:
-                    m = re.match(r'channel-group ([0-9]+) .*', line)
-                    if m:
-                        chnMemberId = m.group(1)
-                        key = f"Port-Channel{chnMemberId}"
-                        out.setdefault(key, {})
-                        out[key].setdefault('channel-member', [])
-                        out[key]['channel-member'].append(self.getSystemValidPortName(interfaceSt))
-        return out
-
-    @staticmethod
-    def getinfo(ansibleOut):
-        """Get Info. So far mainly mac address is used"""
-        return {'mac': ansibleOut['systemMacAddress']}
-
-    def getlldpneighbors(self, ansibleOut):
-        """Get LLDP Neighbors information"""
-        out = {}
-        for localPort, neighbors in ansibleOut['lldpNeighbors'].items():
-            if not neighbors['lldpNeighborInfo']:
-                # Port does not have any neighbors
-                continue
-            if len(neighbors['lldpNeighborInfo']) > 1:
-                continue
-            lldpInfo = neighbors['lldpNeighborInfo'][0]
-            tmpEntry = {'local_port_id': localPort}
-            # Mac comes like: 4c76.25e8.44c0
-            # We need to have: 4c:76:25:e8:44:c0
-            mac = lldpInfo['chassisId'].replace('.', '')
-            split_mac = [mac[index : index + 2] for index in range(0, len(mac), 2)]
-            mac = ":".join(split_mac)
-            tmpEntry['remote_chassis_id'] = mac
-            if 'systemName' in lldpInfo:
-                tmpEntry['remote_system_name'] = lldpInfo['systemName']
-            if lldpInfo['neighborInterfaceInfo']['interfaceIdType'] == 'macAddress':
-                # Means this port goes to server itself
-                tmpEntry['remote_port_id'] = self.getSystemValidPortName(lldpInfo['neighborInterfaceInfo']['interfaceDescription'])
-            elif lldpInfo['neighborInterfaceInfo']['interfaceIdType'] == 'interfaceName':
-                # Means this port goes to another switch
-                tmpEntry['remote_port_id'] = self.getSystemValidPortName(lldpInfo['neighborInterfaceInfo']['interfaceId'])
-            elif lldpInfo['neighborInterfaceInfo']['interfaceIdType'] == 'local':
-                tmpEntry['remote_port_id'] = self.getSystemValidPortName(lldpInfo['neighborInterfaceInfo']['interfaceDescription'])
-            out[localPort] = tmpEntry
-        return out
-
-    @staticmethod
-    def __getRouting(ansibleOut):
-        """Get Routing Info from Arista EOS Devices"""
-        out = []
-        for vrf, routes in ansibleOut.get('vrfs', {}).items():
-            for route, routed in routes.get('routes', {}).items():
-                for routestats in routed.get('vias', []):
-                    nRoute = {}
-                    if vrf != 'default':
-                        nRoute['vrf'] = vrf
-                    nRoute['to'] = route
-                    if 'nexthopAddr' in routestats:
-                        nRoute['from'] = routestats['nexthopAddr']
-                    if 'interface' in routestats:
-                        nRoute['intf'] = routestats['interface']
-                    out.append(nRoute)
-        return out
-
-    def getIPv4Routing(self, ansibleOut):
-        """Get IPv4 Routing information"""
-        return self.__getRouting(ansibleOut)
-
-
-    def getIPv6Routing(self, ansibleOut):
-        """Get IPv6 Routing information"""
-        return self.__getRouting(ansibleOut)
 
 MODULE = AristaEOS