From 29143b0d8a715f0f4f7585e96f1c42590cdc4e37 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Tue, 25 Jun 2024 23:02:23 -0400 Subject: [PATCH 1/5] changes to support child interfaces for DedicatedPort --- fim/graph/abc_property_graph.py | 75 +++++++++- fim/graph/data/graph_validation_rules.json | 4 +- fim/graph/slices/abc_asm.py | 11 ++ fim/slivers/interface_info.py | 54 ++++++- fim/user/interface.py | 84 ++++++++++- fim/user/topology.py | 51 +++++++ test/slice_topology_test.py | 161 +++++++++++++++++++-- test/sliver_json_test.py | 14 +- 8 files changed, 430 insertions(+), 24 deletions(-) diff --git a/fim/graph/abc_property_graph.py b/fim/graph/abc_property_graph.py index 00cf1a2..df4d724 100644 --- a/fim/graph/abc_property_graph.py +++ b/fim/graph/abc_property_graph.py @@ -41,7 +41,7 @@ from fim.slivers.capacities_labels import Capacities, Labels, ReservationInfo, \ StructuralInfo, CapacityHints, Location, Flags from fim.slivers.delegations import Delegations, DelegationType -from fim.slivers.interface_info import InterfaceSliver, InterfaceInfo +from fim.slivers.interface_info import InterfaceSliver, InterfaceInfo, InterfaceType from fim.slivers.base_sliver import BaseSliver from fim.slivers.network_node import NodeSliver, CompositeNodeSliver from fim.slivers.network_link import NetworkLinkSliver @@ -335,6 +335,18 @@ def get_nodes_on_shortest_path(self, *, node_a: str, node_z: str, rel: str = Non :return: """ + def get_nodes_on_path_with_hops(self, *, node_a: str, node_z: str, hops: List[str], cut_off: int = 100) -> List: + """ + Get a list of node ids that lie on a path between two nodes with the specified hops. Return empty + list if no path can be found. Optionally specify the type of relationship that path + should consist of. + :param node_a: Starting node ID. + :param node_z: Ending node ID. + :param hops: List of hops that must be present in the path. + :param cut_off: Optional Depth to stop the search. Only paths of length <= cutoff are returned. + :return: Path with specified hops and no loops exists, empty list otherwise. + """ + @abstractmethod def get_first_neighbor(self, *, node_id: str, rel: str, node_label: str) -> List: """ @@ -665,7 +677,7 @@ def sliver_to_dict(sliver) -> Dict[str, Any]: if sliver.network_service_info is not None: nss = list() - for ns in sliver.network_service_info.list_network_services(): + for ns in sliver.network_service_info.list_services(): nss.append(ABCPropertyGraph.sliver_to_dict(ns)) d['network_services'] = nss @@ -693,6 +705,13 @@ def sliver_to_dict(sliver) -> Dict[str, Any]: d = ABCPropertyGraph.link_sliver_to_graph_properties_dict(sliver) elif type(sliver) == InterfaceSliver: d = ABCPropertyGraph.interface_sliver_to_graph_properties_dict(sliver) + # now add deep sliver stuff + # interfaces + if sliver.interface_info is not None: + ii = list() + for i in sliver.interface_info.list_interfaces(): + ii.append(ABCPropertyGraph.sliver_to_dict(i)) + d['interfaces'] = ii else: raise PropertyGraphQueryException(msg=f'JSON Conversion for type {type(sliver)} is not supported.', graph_id=None, node_id=None) @@ -825,6 +844,14 @@ def interface_sliver_from_graph_properties_dict(d: Dict[str, str]) -> InterfaceS isl = InterfaceSliver() ABCPropertyGraph.set_base_sliver_properties_from_graph_properties_dict(isl, d) isl.set_properties(peer_labels=Labels.from_json(d.get(ABCPropertyGraph.PROP_PEER_LABELS, None))) + # find interfaces and attach + ifs = d.get('interfaces', None) + if ifs is not None and len(ifs) > 0: + ifi = InterfaceInfo() + for i in ifs: + ifsl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(i) + ifi.add_interface(ifsl) + isl.interface_info = ifi return isl def build_deep_node_sliver(self, *, node_id: str) -> NodeSliver: @@ -853,6 +880,7 @@ def build_deep_node_sliver(self, *, node_id: str) -> NodeSliver: nss = self.get_first_neighbor(node_id=node_id, rel=ABCPropertyGraph.REL_HAS, node_label=ABCPropertyGraph.CLASS_NetworkService) + if nss is not None and len(nss) > 0: nsi = NetworkServiceInfo() for s in nss: @@ -906,11 +934,14 @@ def build_deep_ns_sliver(self, *, node_id: str) -> NetworkServiceSliver: # find interfaces and attach ifs = self.get_first_neighbor(node_id=node_id, rel=ABCPropertyGraph.REL_CONNECTS, node_label=ABCPropertyGraph.CLASS_ConnectionPoint) + if ifs is not None and len(ifs) > 0: ifi = InterfaceInfo() for i in ifs: - _, iprops = self.get_node_properties(node_id=i) - ifsl = self.interface_sliver_from_graph_properties_dict(iprops) + # Take child interfaces into account + ifsl = self.build_deep_interface_sliver(node_id=i) + #_, iprops = self.get_node_properties(node_id=i) + #ifsl = self.interface_sliver_from_graph_properties_dict(iprops) ifi.add_interface(ifsl) nss.interface_info = ifi return nss @@ -988,6 +1019,18 @@ def build_deep_interface_sliver(self, *, node_id: str) -> InterfaceSliver: msg="Node is not of class Interface") # create top-level sliver isl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(props) + + # find interfaces and attach + if isl.get_type() == InterfaceType.DedicatedPort and not isl.interface_info: + ifs = self.get_first_neighbor(node_id=node_id, rel=ABCPropertyGraph.REL_CONNECTS, + node_label=ABCPropertyGraph.CLASS_ConnectionPoint) + if ifs is not None and len(ifs) > 0: + ifi = InterfaceInfo() + for i in ifs: + _, iprops = self.get_node_properties(node_id=i) + ifsl = self.interface_sliver_from_graph_properties_dict(iprops) + ifi.add_interface(ifsl) + isl.interface_info = ifi return isl @staticmethod @@ -999,6 +1042,15 @@ def build_deep_interface_sliver_from_dict(*, props: Dict[str, Any]) -> Interface """ # create top-level sliver isl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(props) + + # find interfaces and attach + ifs = props.get('interfaces', None) + if ifs is not None and len(ifs) > 0: + ifi = InterfaceInfo() + for i in ifs: + ifsl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(i) + ifi.add_interface(ifsl) + isl.interface_info = ifi return isl def build_deep_link_sliver(self, *, node_id: str) -> NetworkLinkSliver: @@ -1257,6 +1309,21 @@ def get_all_ns_or_link_connection_points(self, link_id: str) -> List[str]: return self.get_first_neighbor(node_id=link_id, rel=ABCPropertyGraph.REL_CONNECTS, node_label=ABCPropertyGraph.CLASS_ConnectionPoint) + def get_all_child_connection_points(self, interface_id: str) -> List[str]: + """ + Get child interfaces attached to a Dedicated Interface + :param interface_id: + :return: + """ + assert interface_id is not None + # check this is a link + labels, parent_props = self.get_node_properties(node_id=interface_id) + if ABCPropertyGraph.CLASS_ConnectionPoint not in labels: + raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=interface_id, + msg="Node type is not ConnectionPoint") + return self.get_first_neighbor(node_id=interface_id, rel=ABCPropertyGraph.REL_CONNECTS, + node_label=ABCPropertyGraph.CLASS_ConnectionPoint) + def get_all_node_or_component_connection_points(self, parent_node_id: str) -> List[str]: """ Get a list of interfaces attached via network services diff --git a/fim/graph/data/graph_validation_rules.json b/fim/graph/data/graph_validation_rules.json index f8678e3..9b35708 100644 --- a/fim/graph/data/graph_validation_rules.json +++ b/fim/graph/data/graph_validation_rules.json @@ -20,8 +20,8 @@ "msg": "All Components must be of type SharedNIC, SmartNIC, GPU, FPGA, Storage or NVME" }, { - "rule": "MATCH (n:ConnectionPoint {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"AccessPort\", \"TrunkPort\", \"ServicePort\", \"DedicatedPort\", \"SharedPort\", \"vInt\", \"FacilityPort\", \"StitchPort\"])", - "msg": "All ConnectionPoints must be of type AccessPort, TrunkPort, ServicePort, DedicatedPort, SharedPort, vInt, FacilityPort or StitchPort" + "rule": "MATCH (n:ConnectionPoint {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"AccessPort\", \"TrunkPort\", \"ServicePort\", \"DedicatedPort\", \"SharedPort\", \"vInt\", \"FacilityPort\", \"SubInterface\", \"StitchPort\"])", + "msg": "All ConnectionPoints must be of type AccessPort, TrunkPort, ServicePort, DedicatedPort, SharedPort, vInt, FacilityPort, SubInterface or StitchPort" }, { "rule": "MATCH (n:NetworkService {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [ \"P4\", \"OVS\", \"MPLS\", \"VLAN\", \"L2Path\", \"L2Bridge\", \"L2PTP\", \"L2STS\", \"FABNetv4\", \"FABNetv6\", \"FABNetv4Ext\", \"FABNetv6Ext\", \"L3VPN\", \"PortMirror\"])", diff --git a/fim/graph/slices/abc_asm.py b/fim/graph/slices/abc_asm.py index 669e2af..bdbb9dd 100644 --- a/fim/graph/slices/abc_asm.py +++ b/fim/graph/slices/abc_asm.py @@ -189,3 +189,14 @@ def find_connection_point_by_name(self, *, parent_node_id: str, iname: str) -> s raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=None, msg=f"Unable to find ConnectionPoint with name {iname}") + def find_child_connection_point_by_name(self, *, parent_node_id: str, iname: str) -> str: + + assert iname is not None + + if_id_list = self.get_all_child_connection_points(interface_id=parent_node_id) + for cid in if_id_list: + _, cprops = self.get_node_properties(node_id=cid) + if cprops[ABCPropertyGraph.PROP_NAME] == iname: + return cid + raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=None, + msg=f"Unable to find ConnectionPoint with name {iname}") diff --git a/fim/slivers/interface_info.py b/fim/slivers/interface_info.py index 1b08d6e..f0cdce8 100644 --- a/fim/slivers/interface_info.py +++ b/fim/slivers/interface_info.py @@ -28,7 +28,7 @@ import enum from .base_sliver import BaseSliver -from .topology_diff import TopologyDiff +from .topology_diff import TopologyDiff, WhatsModifiedFlag, TopologyDiffTuple, TopologyDiffModifiedTuple from .capacities_labels import Labels @@ -44,6 +44,7 @@ class InterfaceType(enum.Enum): vInt = enum.auto() StitchPort = enum.auto() FacilityPort = enum.auto() + SubInterface = enum.auto() def help(self) -> str: return 'An ' + self.name @@ -64,6 +65,7 @@ def __init__(self): super().__init__() # note that these are used in ASMs, not delegateable self.peer_labels = None + self.interface_info = None def set_peer_labels(self, lab: Labels) -> None: assert(lab is None or isinstance(lab, Labels)) @@ -81,7 +83,55 @@ def type_from_str(ntype: str) -> InterfaceType: return t def diff(self, other_sliver) -> TopologyDiff or None: - raise RuntimeError('Not implemented') + if not other_sliver: + return None + + super().diff(other_sliver) + + ifs_added = set() + ifs_removed = set() + ifs_modified = list() + + # see if we ourselves have modified properties + self_modified = list() + self_modified_flags = self.prop_diff(other_sliver) + if self.prop_diff(other_sliver) != WhatsModifiedFlag.NONE: + self_modified.append((self, self_modified_flags)) + + if self.interface_info and other_sliver.interface_info: + diff_comps = self._dict_diff(self.interface_info.interfaces, + other_sliver.interface_info.interfaces) + ifs_added = set(diff_comps['added'].values()) + ifs_removed = set(diff_comps['removed'].values()) + # there are interfaces in common, so we check if they have been modified + ifs_common = self._dict_common(self.interface_info.interfaces, + other_sliver.interface_info.interfaces) + for iA in ifs_common.values(): + iB = other_sliver.interface_info.get_interface(iA.resource_name) + # compare properties + flag = iA.prop_diff(iB) + if flag != WhatsModifiedFlag.NONE: + ifs_modified.append((iA, flag)) + + if not self.interface_info and other_sliver.interface_info: + ifs_added = set(other_sliver.interface_info.interfaces.values()) + + if self.interface_info and not other_sliver.interface_info: + ifs_removed = set(self.interface_info.interfaces.values()) + + if len(self_modified) > 0 or len(ifs_added) > 0 or len(ifs_removed) > 0 or len(ifs_modified) > 0: + return TopologyDiff(added=TopologyDiffTuple(components=set(), services=set(), interfaces=ifs_added, + nodes=set()), + removed=TopologyDiffTuple(components=set(), services=set(), + interfaces=ifs_removed, nodes=set()), + modified=TopologyDiffModifiedTuple( + nodes=list(), + components=list(), + services=self_modified, + interfaces=ifs_modified) + ) + else: + return None class InterfaceInfo: diff --git a/fim/user/interface.py b/fim/user/interface.py index aee63da..e80223c 100644 --- a/fim/user/interface.py +++ b/fim/user/interface.py @@ -33,6 +33,7 @@ from ..slivers.interface_info import InterfaceType, InterfaceSliver from ..graph.abc_property_graph import ABCPropertyGraph from ..slivers.capacities_labels import Labels +from ..view_only_dict import ViewOnlyDict class Interface(ModelElement): @@ -77,19 +78,98 @@ def __init__(self, *, name: str, node_id: str = None, topo: Any, sliver.set_properties(**kwargs) self.topo.graph_model.add_interface_sliver(parent_node_id=parent_node_id, interface=sliver) + self._interfaces = list() else: assert node_id is not None super().__init__(name=name, node_id=node_id, topo=topo) if check_existing and not self.topo.graph_model.check_node_name(node_id=node_id, name=name, label=ABCPropertyGraph.CLASS_ConnectionPoint): raise TopologyException(f"Interface with this id {node_id} and name {name} doesn't exist") + # collect a list of interface nodes it attaches to for DedicatedPorts only + self._interfaces = list() + if self.type == InterfaceType.DedicatedPort: + interface_list = self.topo.graph_model.get_all_child_connection_points(interface_id=self.node_id) + name_id_tuples = list() + # need to look up their names - a bit inefficient, need to think about this /ib + for iff in interface_list: + _, props = self.topo.graph_model.get_node_properties(node_id=iff) + name_id_tuples.append((props[ABCPropertyGraph.PROP_NAME], iff)) + self._interfaces = [Interface(node_id=tup[1], topo=topo, name=tup[0]) for tup in name_id_tuples] @property def type(self): return self.get_property('type') if self.__dict__.get('topo', None) is not None else None - def add_child_interface(self): - raise TopologyException("Not implemented") + def add_child_interface(self, *, name: str, node_id: str = None, **kwargs): + """ + Add an interface to network service + :param name: + :param node_id: + :param itype: interface type e.g. TrunkPort, AccessPort or VINT + :param kwargs: additional parameters + :return: + """ + assert name is not None + assert self.type is InterfaceType.DedicatedPort + + # check uniqueness + all_names = [n.name for n in self._interfaces] + if name in all_names: + raise TopologyException(f'Sub Interface {name} is not unique within an interface') + iff = Interface(name=name, node_id=node_id, parent_node_id=self.node_id, + etype=ElementType.NEW, topo=self.topo, itype=InterfaceType.SubInterface, + **kwargs) + + self._interfaces.append(iff) + return iff + + def remove_child_interface(self, *, name: str) -> None: + """ + Remove an ServicePort interface from the network service. + :param name: + :return: + """ + assert name is not None + assert self.type is InterfaceType.DedicatedPort + + # cant use isinstance as it would create circular import dependencies + #if str(self.topo.__class__) == "": + # raise TopologyException("Cannot remove child interface interface from Interface in Experiment topology") + node_id = self.topo.graph_model.find_child_connection_point_by_name(parent_node_id=self.node_id, + iname=name) + # TODO validate if this works + self.topo.graph_model.remove_cp_and_links(node_id=node_id) + + def __list_interfaces(self) -> ViewOnlyDict: + """ + List all interfaces of the network service as a dictionary + :return: + """ + ret = dict() + for intf in self._interfaces: + ret[intf.name] = intf + return ViewOnlyDict(ret) + + def __list_of_interfaces(self) -> tuple: + """ + Return a list of names of interfaces of network service + :return: + """ + return tuple(self._interfaces) + + @property + def interface_list(self): + """ + List of names of service interfaces + """ + return self.__list_of_interfaces() + + @property + def interfaces(self): + """ + Dictionary name->Interface for all interfaces + """ + return self.__list_interfaces() @property def peer_labels(self): diff --git a/fim/user/topology.py b/fim/user/topology.py index e0d7f67..ff0d969 100644 --- a/fim/user/topology.py +++ b/fim/user/topology.py @@ -108,6 +108,14 @@ def get_parent_element(self, e: ModelElement) -> ModelElement or None: # interfaces have NSs as parents if isinstance(e, Interface): + if e.type == InterfaceType.SubInterface: + node_name, node_id = self.graph_model.get_parent(node_id=e.node_id, + rel=ABCPropertyGraph.REL_CONNECTS, + parent=ABCPropertyGraph.CLASS_ConnectionPoint) + if node_id is None: + raise TopologyException(f'Interface {e} has no parent') + return Interface(name=node_name, node_id=node_id, topo=self) + node_name, node_id = self.graph_model.get_parent(node_id=e.node_id, rel=ABCPropertyGraph.REL_CONNECTS, parent=ABCPropertyGraph.CLASS_NetworkService) @@ -273,6 +281,49 @@ def remove_facility(self, *, name: str): raise TopologyException(f'{name} is not a Facility node, cannot remove.') self.remove_node(name) + def add_switch(self, *, name: str, node_id: str = None, site: str, + nstype: ServiceType = ServiceType.P4, nslabels: Labels or None = None, + nports: int = 8, portlabels: Labels or None = None, + portcapacities: Capacities or None = None, + **kwargs) -> Node: + """ + Add a switch (P4 type by default) with some number of ports (8 by default) + all given names and label local_names 'p1'-'p8'. + :param name: + :param node_id: + :param site: + :param nstype: network service type (defaults to P4) + :param nslabels: additional labels for switch network service + :param nports: number of ports (defaults to 8) + :param portlabels: labels to be added to each port (otherwise overridden with default) + :param portcapacities: capacities to be added to each port (otherwise overridden with default) + :param kwargs: pass additional parameters to add node (e.g. model) + """ + switch = self.add_node(name=name, node_id=node_id, site=site, ntype=NodeType.Switch) + switch_ns = switch.add_network_service(name=name + '-ns', + node_id=node_id + '-ns' if node_id else None, + nstype=nstype, labels=nslabels) + # name them 'p1'-'p8' + for i in range(1, nports + 1): + labels = Labels(local_name=f'p{i}') + # 100G port + capacities = Capacities(bw=100) + switch_i = switch_ns.add_interface(name=f'p{i}', node_id=node_id + f'-int{i}' if node_id else None, + itype=InterfaceType.DedicatedPort, + labels=portlabels if portlabels else labels, + capacities=portcapacities if portcapacities else capacities) + return switch + + def remove_switch(self, *, name: str): + """ + Remove a switch and associated network service and interfaces, disconnecting it from a + service as appropriate. Same as removing a node. + """ + fac = self._get_node_by_name(name) + if fac.type != NodeType.Switch: + raise TopologyException(f'{name} is not a Switch node, cannot remove.') + self.remove_node(name) + def add_link(self, *, name: str, node_id: str = None, ltype: LinkType, interfaces: List[Interface], technology: str = None, **kwargs) -> Link: diff --git a/test/slice_topology_test.py b/test/slice_topology_test.py index e117252..31cc3d4 100644 --- a/test/slice_topology_test.py +++ b/test/slice_topology_test.py @@ -24,7 +24,7 @@ class SliceTest(unittest.TestCase): neo4j = {"url": "neo4j://0.0.0.0:7687", "user": "neo4j", "pass": "password", - "import_host_dir": "neo4j/imports/", + "import_host_dir": "neo4j/imports", "import_dir": "/imports"} def setUp(self) -> None: @@ -258,9 +258,6 @@ def testNetworkServices(self): nic1 = n1.components['nic1'] nic2 = n2.components['nic2'] - p1 = nic2.interfaces['nic2-p1'] - p2 = nic2.interfaces['nic2-p2'] - cap = f.Capacities(bw=50, unit=1) nic1.capacities = cap lab = f.Labels(ipv4="192.168.1.12") @@ -280,6 +277,7 @@ def testNetworkServices(self): s1 = self.topo.add_network_service(name='s1', nstype=f.ServiceType.L2STS, interfaces=[n1.interface_list[0], n2.interface_list[0], n3.interface_list[0]]) + p1 = n1.interface_list[0] # facilities fac1 = self.topo.add_facility(name='RENCI-DTN', site='RENC', capacities=f.Capacities(bw=10), @@ -315,11 +313,10 @@ def testNetworkServices(self): s1p = self.topo.network_services['s1'] print(f'S1 has these interfaces: {s1p.interface_list}') - - s1.disconnect_interface(interface=p1) - - print(f'S1 has these interfaces: {s1.interface_list}') - self.assertEqual(len(s1.interface_list), 2) + print(f'Disconnecting {p1} from S1') + s1p.disconnect_interface(interface=p1) + print(f'Now S1 has these interfaces: {s1p.interface_list}') + self.assertEqual(len(s1p.interface_list), 2) # validate the topology self.topo.validate() @@ -510,6 +507,7 @@ def testBasicOneSiteSliceNeo4jImp(self): interfaces=topo.interface_list) topo.serialize(file_name='single-site-neo4jimp.graphml') topo.validate() + os.unlink('single-site-neo4jimp.graphml') def testBasicTwoSiteSlice(self): # create a basic slice and export to GraphML and JSON @@ -599,9 +597,11 @@ def testL3Service(self): asm_graph.validate_graph() self.n4j_imp.delete_all_graphs() + ''' def testL3ServiceFail(self): """ Test validaton of max 1 L3 service per site of a given type + No longer a valid test case as validation constraint number of Fabnets per site was removed """ self.topo.add_node(name='n1', site='RENC', ntype=f.NodeType.VM) self.topo.add_node(name='n2', site='RENC') @@ -631,6 +631,112 @@ def testL3ServiceFail(self): # as self.topo.validate() but is much faster asm_graph.validate_graph() self.n4j_imp.delete_all_graphs() + ''' + + def testL3ServiceFail2(self): + """ + Test validaton of L3 service required per site + """ + self.topo.add_node(name='n1', site='RENC', ntype=f.NodeType.VM) + self.topo.add_node(name='n2', site='RENC') + self.topo.add_node(name='n3', site='UKY') + self.topo.nodes['n1'].add_component(model_type=f.ComponentModelType.SharedNIC_ConnectX_6, name='nic1') + self.topo.nodes['n2'].add_component(model_type=f.ComponentModelType.SmartNIC_ConnectX_6, name='nic1') + self.topo.nodes['n3'].add_component(model_type=f.ComponentModelType.SmartNIC_ConnectX_5, name='nic1') + + s1 = self.topo.add_network_service(name='globalL3', nstype=f.ServiceType.FABNetv4, + interfaces=[self.topo.nodes['n1'].interface_list[0], + self.topo.nodes['n2'].interface_list[0], + self.topo.nodes['n3'].interface_list[0]]) + + # site property is set automagically by validate + with self.assertRaises(TopologyException): + self.topo.validate() + + slice_graph = self.topo.serialize() + + # Import it in the neo4j as ASM + generic_graph = self.n4j_imp.import_graph_from_string(graph_string=slice_graph) + asm_graph = Neo4jASMFactory.create(generic_graph) + # the following validation just uses cypher or networkx_query and is not as capable + # as self.topo.validate() but is much faster + asm_graph.validate_graph() + self.n4j_imp.delete_all_graphs() + + def testSubInterfaces_1(self): + n1 = self.topo.add_node(name='Node1', site='RENC') + n1_nic1 = n1.add_component(ctype=f.ComponentType.SmartNIC, model='ConnectX-6', name='nic1') + n1_nic1_interface1 = n1_nic1.interface_list[0] + n1_nic1_interface1.add_child_interface(name="child1", labels=Labels(vlan="100")) + + self.assertTrue(len(n1_nic1_interface1.interface_list) != 0) + self.assertEqual(n1_nic1_interface1.interface_list[0].type, f.InterfaceType.SubInterface) + + self.topo.validate() + + slice_graph = self.topo.serialize() + # Import it in the neo4j as ASM + generic_graph = self.n4j_imp.import_graph_from_string(graph_string=slice_graph) + asm_graph = Neo4jASMFactory.create(generic_graph) + asm_graph.validate_graph() + for n in asm_graph.get_all_network_nodes(): + node_sliver = asm_graph.build_deep_node_sliver(node_id=n) + print(node_sliver) + for c in node_sliver.attached_components_info.devices.values(): + print(f"\t{c}") + for ns in c.network_service_info.network_services.values(): + print(f"\t\t{ns}") + for ifs in ns.interface_info.interfaces.values(): + print(f"\t\t\t{ifs}") + if "p1" in ifs.get_name(): + self.assertTrue(ifs.interface_info is not None) + for cifs in ifs.interface_info.interfaces.values(): + self.assertEqual(cifs.get_type(), f.InterfaceType.SubInterface) + print(f"\t\t\t\t{cifs}") + + self.n4j_imp.delete_all_graphs() + + def testSubInterfaces(self): + n1 = self.topo.add_node(name='Node1', site='RENC') + n2 = self.topo.add_node(name='Node2', site='RENC') + n1_nic1 = n1.add_component(ctype=f.ComponentType.SmartNIC, model='ConnectX-6', name='nic1') + n1_nic1_interface1 = n1_nic1.interface_list[0] + n1_nic1_interface1.add_child_interface(name="child1", labels=Labels(vlan="100")) + + self.assertTrue(len(n1_nic1_interface1.interface_list) != 0) + self.assertEqual(n1_nic1_interface1.interface_list[0].type, f.InterfaceType.SubInterface) + + n2_nic1 = n2.add_component(ctype=f.ComponentType.SmartNIC, model='ConnectX-6', name='nic1') + n2_nic1_interface1 = n2_nic1.interface_list[0] + n2_nic1_interface1.add_child_interface(name="child1", labels=Labels(vlan="200")) + + self.assertTrue(len(n2_nic1_interface1.interface_list) != 0) + self.assertEqual(n2_nic1_interface1.interface_list[0].type, f.InterfaceType.SubInterface) + + ns = self.topo.add_network_service(name='ns1', nstype=f.ServiceType.L2Bridge, + interfaces=[n2_nic1_interface1.interface_list[0], + n1_nic1_interface1.interface_list[0]]) + self.assertTrue(len(ns.interface_list) != 0) + self.topo.validate() + + slice_graph = self.topo.serialize() + print(f"SubInterfaces: {slice_graph}") + + print(self.topo.network_services) + + self.topo.remove_network_service(name='ns1') + self.topo.validate() + + slice_graph_after_removal = self.topo.serialize() + print(f"SubInterfaces after removing network service: {slice_graph_after_removal}") + + # Import it in the neo4j as ASM + generic_graph = self.n4j_imp.import_graph_from_string(graph_string=slice_graph) + asm_graph = Neo4jASMFactory.create(generic_graph) + asm_graph.validate_graph() + print(self.topo.network_services) + + self.n4j_imp.delete_all_graphs() def testPortMirrorService(self): t = self.topo @@ -706,7 +812,6 @@ def testFPGAComponent(self): slice_graph = t.serialize(file_name='fpga_slice.graphml') slice_graph = t.serialize() - # Import it in the neo4j as ASM generic_graph = self.n4j_imp.import_graph_from_string(graph_string=slice_graph) asm_graph = Neo4jASMFactory.create(generic_graph) @@ -723,9 +828,9 @@ def testMultiConnectedFacility(self): n2.add_component(name='nic1', model_type=ComponentModelType.SmartNIC_ConnectX_6) # add facility - fac1 = self.topo.add_facility(name='RENCI-DTN', site='RENC', - interfaces=[('to_mass', f.Labels(vlan='100'), f.Capacities(bw=10)), - ('to_renc', f.Labels(vlan='101'), f.Capacities(bw=1))]) + fac1 = t.add_facility(name='RENCI-DTN', site='RENC', + interfaces=[('to_mass', f.Labels(vlan='100'), f.Capacities(bw=10)), + ('to_renc', f.Labels(vlan='101'), f.Capacities(bw=1))]) t.add_network_service(name='ns1', nstype=ServiceType.L2PTP, interfaces=[n1.interface_list[0], fac1.interface_list[0]]) @@ -741,6 +846,36 @@ def testMultiConnectedFacility(self): self.n4j_imp.delete_all_graphs() + def testP4Switch(self): + t = self.topo + + n1 = t.add_node(name='n1', site='MASS') + n1.add_component(name='nic1', model_type=ComponentModelType.SmartNIC_ConnectX_6) + n2 = t.add_node(name='n2', site='RENC') + n2.add_component(name='nic1', model_type=ComponentModelType.SmartNIC_ConnectX_6) + n3 = t.add_node(name='n3', site='MASS') + n3.add_component(name='nic1', model_type=ComponentModelType.SmartNIC_ConnectX_6) + + # add P4 switch at another site + sw = t.add_switch(name='p4switch', site='STAR') + + # instead of sw.interfaces['p1'] you can also use sw.list_interfaces[0] however + # for p4 switches referencing by name may be more appropriate for the users + t.add_network_service(name='ns1', nstype=ServiceType.L2PTP, + interfaces=[n1.interface_list[0], sw.interfaces['p1']]) + t.add_network_service(name='ns2', nstype=ServiceType.L2PTP, + interfaces=[n2.interface_list[0], sw.interfaces['p2']]) + t.add_network_service(name='ns3', nstype=ServiceType.L2PTP, + interfaces=[n3.interface_list[0], sw.interfaces['p3']]) + + t.validate() + + t.serialize(file_name='p4_switch_slice.graphml') + + self.n4j_imp.delete_all_graphs() + + os.unlink('p4_switch_slice.graphml') + def testL3VPNWithCloudService(self): t = self.topo diff --git a/test/sliver_json_test.py b/test/sliver_json_test.py index 3452e17..cb3e339 100644 --- a/test/sliver_json_test.py +++ b/test/sliver_json_test.py @@ -1,3 +1,4 @@ +import json import unittest import fim.user as f @@ -11,7 +12,8 @@ def testNodeAndServiceSlivers(self): t = f.ExperimentTopology() t.add_node(name='n1', site='RENC') t.nodes['n1'].capacities = f.Capacities(core=1) - t.nodes['n1'].add_component(name='c1', ctype=f.ComponentType.SmartNIC, model='ConnectX-6') + c = t.nodes['n1'].add_component(name='c1', ctype=f.ComponentType.SmartNIC, model='ConnectX-6') + c.interface_list[0].add_child_interface(name="child1", labels=f.Labels(vlan="100")) d = ABCPropertyGraph.sliver_to_dict(t.nodes['n1'].get_sliver()) t.add_node(name='n2', site='RENC') t.nodes['n2'].add_component(name='c2', ctype=f.ComponentType.SmartNIC, model='ConnectX-6') @@ -26,6 +28,16 @@ def testNodeAndServiceSlivers(self): self.assertEqual(ns1.capacities.core, 1) self.assertEqual(len(ns1.attached_components_info.list_devices()), 1) + for c in ns1.attached_components_info.list_devices(): + print(c) + for ns in c.network_service_info.network_services.values(): + for ifs in ns.interface_info.interfaces.values(): + print(ifs) + if "p1" in ifs.get_name(): + self.assertTrue(ifs.interface_info is not None) + for cifs in ifs.interface_info.interfaces.values(): + print(cifs) + self.assertEqual(len(ns2.attached_components_info.list_devices()), 2) self.assertEqual(len(nss1.interface_info.list_interfaces()), 4) inames = [i.resource_name for i in nss1.interface_info.list_interfaces()] From 67e7c584759b150b45dd97186de723429be50e85 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Wed, 26 Jun 2024 14:56:01 -0400 Subject: [PATCH 2/5] updated modify to handler modification of components w.r.t sub interfaces --- fim/slivers/network_node.py | 9 +++++++++ fim/slivers/network_service.py | 6 ++++++ fim/slivers/topology_diff.py | 1 + test/modify_test.py | 9 ++++++++- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/fim/slivers/network_node.py b/fim/slivers/network_node.py index a3e5454..0967690 100644 --- a/fim/slivers/network_node.py +++ b/fim/slivers/network_node.py @@ -28,6 +28,7 @@ from recordclass import recordclass from fim.slivers.capacities_labels import Location +from .attached_components import ComponentType from .base_sliver import BaseSliver from .topology_diff import TopologyDiff, TopologyDiffTuple, TopologyDiffModifiedTuple, WhatsModifiedFlag from fim.slivers.maintenance_mode import MaintenanceInfo @@ -191,6 +192,14 @@ def diff(self, other_sliver) -> TopologyDiff or None: cB = other_sliver.attached_components_info.get_device(cA.resource_name) # compare properties flag = cA.prop_diff(cB) + + # compare child interfaces + if cA.get_type() == ComponentType.SmartNIC: + cAns = list(cA.network_service_info.network_services.values())[0] + cBns = list(cB.network_service_info.network_services.values())[0] + if cAns.diff(cBns): + flag |= WhatsModifiedFlag.SUB_INTERFACES + if flag != WhatsModifiedFlag.NONE: comp_modified.append((cA, flag)) diff --git a/fim/slivers/network_service.py b/fim/slivers/network_service.py index 77bba5e..464532a 100644 --- a/fim/slivers/network_service.py +++ b/fim/slivers/network_service.py @@ -383,6 +383,12 @@ def diff(self, other_sliver) -> TopologyDiff or None: iB = other_sliver.interface_info.get_interface(iA.resource_name) # compare properties flag = iA.prop_diff(iB) + + if iA.get_type() == InterfaceType.DedicatedPort: + if iA.diff(iB): + print("Added the child interfaces") + flag |= WhatsModifiedFlag.SUB_INTERFACES + if flag != WhatsModifiedFlag.NONE: ifs_modified.append((iA, flag)) diff --git a/fim/slivers/topology_diff.py b/fim/slivers/topology_diff.py index 4c695f5..39245a1 100644 --- a/fim/slivers/topology_diff.py +++ b/fim/slivers/topology_diff.py @@ -40,6 +40,7 @@ class WhatsModifiedFlag(Flag): LABELS = auto() CAPACITIES = auto() USER_DATA = auto() + SUB_INTERFACES = auto() @dataclasses.dataclass diff --git a/test/modify_test.py b/test/modify_test.py index a172b88..ce28195 100644 --- a/test/modify_test.py +++ b/test/modify_test.py @@ -63,6 +63,13 @@ def modifyActions(self): """ Define modify actions on the initial topo """ + # add a sub interface to nic1 + nA = self.topoB.nodes["NodeA"] + nic1 = nA.components["nic1"] + nic1_interface = nic1.interface_list[0] + child1 = nic1_interface.add_child_interface(name="nic1-child1", labels=Labels(vlan="100")) + self.diff.added.interfaces.add(child1) + # # add a node with components, components won't show up as 'added' nB = self.topoB.add_node(name='NodeB', site='UKY') @@ -330,7 +337,7 @@ def testSliverDiffs(self): self.assertEqual(len(diff.modified.nodes), 1) self.assertEqual(len(diff.modified.services), 0) self.assertEqual(len(diff.modified.interfaces), 0) - self.assertEqual(len(diff.modified.components), 0) + self.assertEqual(len(diff.modified.components), 1) sA1 = self.topoA.network_services['bridge2'] sB1 = self.topoB.network_services['bridge2'] From 0f01ec6bfe323201eb5d550ea56923489b538de3 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 28 Jun 2024 01:09:21 -0400 Subject: [PATCH 3/5] disable adding same vlans for child interfaces --- fim/__init__.py | 2 +- fim/user/interface.py | 15 ++++++++++++++- test/slice_topology_test.py | 17 +++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/fim/__init__.py b/fim/__init__.py index 8a76627..33cbe8a 100644 --- a/fim/__init__.py +++ b/fim/__init__.py @@ -1,5 +1,5 @@ """ FABRIC Information Model library and utilities """ -__VERSION__ = "1.7.0b10" +__VERSION__ = "1.7.0b12" __version__ = __VERSION__ diff --git a/fim/user/interface.py b/fim/user/interface.py index e80223c..ae8a01d 100644 --- a/fim/user/interface.py +++ b/fim/user/interface.py @@ -115,7 +115,20 @@ def add_child_interface(self, *, name: str, node_id: str = None, **kwargs): # check uniqueness all_names = [n.name for n in self._interfaces] if name in all_names: - raise TopologyException(f'Sub Interface {name} is not unique within an interface') + raise TopologyException(f'Sub Interface {name} is not unique within the interface') + + labels = kwargs.get('labels') + if not labels or not labels.vlan: + raise TopologyException(f'Vlan must be specified for Sub Interface within the interface') + + all_vlans = [] + for i in self._interfaces: + if i.labels and i.labels.vlan: + all_vlans.append(i.labels.vlan) + + if labels.vlan in all_vlans: + raise TopologyException(f'Vlan in use by another Sub Interface within the interface') + iff = Interface(name=name, node_id=node_id, parent_node_id=self.node_id, etype=ElementType.NEW, topo=self.topo, itype=InterfaceType.SubInterface, **kwargs) diff --git a/test/slice_topology_test.py b/test/slice_topology_test.py index f4c7aef..b4c9903 100644 --- a/test/slice_topology_test.py +++ b/test/slice_topology_test.py @@ -24,7 +24,7 @@ class SliceTest(unittest.TestCase): neo4j = {"url": "neo4j://0.0.0.0:7687", "user": "neo4j", "pass": "password", - "import_host_dir": "neo4j/imports", + "import_host_dir": "/Users/kthare10/renci/code/fabric/1.6/ControlFramework/neo4j1/imports", "import_dir": "/imports"} def setUp(self) -> None: @@ -667,10 +667,23 @@ def testSubInterfaces_1(self): n1 = self.topo.add_node(name='Node1', site='RENC') n1_nic1 = n1.add_component(ctype=f.ComponentType.SmartNIC, model='ConnectX-6', name='nic1') n1_nic1_interface1 = n1_nic1.interface_list[0] + + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child1") + n1_nic1_interface1.add_child_interface(name="child1", labels=Labels(vlan="100")) - self.assertTrue(len(n1_nic1_interface1.interface_list) != 0) + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child1", labels=Labels(vlan="200")) + + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child2", labels=Labels(vlan="100")) + + n1_nic1_interface1.add_child_interface(name="child2", labels=Labels(vlan="200")) + + self.assertEqual(len(n1_nic1_interface1.interface_list), 2) self.assertEqual(n1_nic1_interface1.interface_list[0].type, f.InterfaceType.SubInterface) + self.assertEqual(n1_nic1_interface1.interface_list[1].type, f.InterfaceType.SubInterface) self.topo.validate() From 62a3aa58964eeb9313d988bda3b775b61ce949d6 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 28 Jun 2024 01:10:43 -0400 Subject: [PATCH 4/5] disable adding same vlans for child interfaces --- test/slice_topology_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/slice_topology_test.py b/test/slice_topology_test.py index b4c9903..a1ae2f2 100644 --- a/test/slice_topology_test.py +++ b/test/slice_topology_test.py @@ -24,7 +24,7 @@ class SliceTest(unittest.TestCase): neo4j = {"url": "neo4j://0.0.0.0:7687", "user": "neo4j", "pass": "password", - "import_host_dir": "/Users/kthare10/renci/code/fabric/1.6/ControlFramework/neo4j1/imports", + "import_host_dir": "neo4j/imports", "import_dir": "/imports"} def setUp(self) -> None: From f153df1153840f966063a408130a25dd3ebbc2f5 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Mon, 1 Jul 2024 12:03:51 -0400 Subject: [PATCH 5/5] add more tests and some fixes for sub interfaces --- fim/graph/abc_property_graph.py | 1 + fim/slivers/capacities_labels.py | 2 +- fim/slivers/network_service.py | 2 +- fim/user/interface.py | 2 +- fim/user/topology.py | 10 +++++++++- test/networkxx_pg_test.py | 2 -- test/slice_topology_test.py | 30 ++++++++++++++++++++++++++++++ test/test_load.py | 2 +- test/zz_neo4j_pg_test.py | 2 +- 9 files changed, 45 insertions(+), 8 deletions(-) diff --git a/fim/graph/abc_property_graph.py b/fim/graph/abc_property_graph.py index 3b1073e..d86a40e 100644 --- a/fim/graph/abc_property_graph.py +++ b/fim/graph/abc_property_graph.py @@ -1175,6 +1175,7 @@ def remove_cp_and_links(self, node_id: str): parents = self.get_first_neighbor(node_id=node_id, rel=ABCPropertyGraph.REL_CONNECTS, node_label=ABCPropertyGraph.CLASS_ConnectionPoint) + for parent in parents: # really should only be one parent interface children = self.get_first_neighbor(node_id=parent, rel=ABCPropertyGraph.REL_CONNECTS, diff --git a/fim/slivers/capacities_labels.py b/fim/slivers/capacities_labels.py index 4eecc4c..7606e92 100644 --- a/fim/slivers/capacities_labels.py +++ b/fim/slivers/capacities_labels.py @@ -617,7 +617,7 @@ def to_latlon(self) -> Tuple[float, float]: raise LocationException(f"Unable to interpret response from OpenStreetmaps for address {self.postal}") self.lat = float(response_json[0]['lat']) - self.lon = (response_json[0]['lon']) + self.lon = float(response_json[0]['lon']) return self.lat, self.lon diff --git a/fim/slivers/network_service.py b/fim/slivers/network_service.py index f6f4742..b171f04 100644 --- a/fim/slivers/network_service.py +++ b/fim/slivers/network_service.py @@ -227,7 +227,7 @@ class NetworkServiceSliver(BaseSliver): num_interfaces=1, num_sites=1, num_instances=NO_LIMIT, desc='A port mirroring service in a FABRIC site.', - required_properties=['mirror_port', 'mirror_vlan', + required_properties=['mirror_port', 'mirror_direction', 'site'], forbidden_properties=['controller_url'], required_interface_types=[]), diff --git a/fim/user/interface.py b/fim/user/interface.py index ae8a01d..e2c3976 100644 --- a/fim/user/interface.py +++ b/fim/user/interface.py @@ -150,7 +150,7 @@ def remove_child_interface(self, *, name: str) -> None: # raise TopologyException("Cannot remove child interface interface from Interface in Experiment topology") node_id = self.topo.graph_model.find_child_connection_point_by_name(parent_node_id=self.node_id, iname=name) - # TODO validate if this works + self.topo.graph_model.remove_cp_and_links(node_id=node_id) def __list_interfaces(self) -> ViewOnlyDict: diff --git a/fim/user/topology.py b/fim/user/topology.py index 73cf314..058e697 100644 --- a/fim/user/topology.py +++ b/fim/user/topology.py @@ -757,6 +757,14 @@ def draw(self, *, file_name: str = None, interactive: bool = False, peer_int_parent = self.get_parent_element(peer_int) if peer_int_parent is None: continue + + # Parent for a sub interface is Interface, get the parent again to get NetworkService + if peer_int.type == InterfaceType.SubInterface: + peer_int_parent = self.get_parent_element(peer_int_parent) + + if peer_int_parent is None: + continue + derived_graph.add_edge(ns.name, peer_int_parent.name) for n in all_node_like: if self.get_owner_node(ns) == n: @@ -772,7 +780,7 @@ def draw(self, *, file_name: str = None, interactive: bool = False, raise TopologyException("This level of detail not yet implemented") def add_port_mirror_service(self, *, name: str, node_id: str = None, - from_interface_name: str, from_interface_vlan: str, to_interface: Interface, + from_interface_name: str, to_interface: Interface, from_interface_vlan: str = None, direction: MirrorDirection = MirrorDirection.Both, **kwargs) -> PortMirrorService: """ diff --git a/test/networkxx_pg_test.py b/test/networkxx_pg_test.py index 5d52ecc..0627a7f 100644 --- a/test/networkxx_pg_test.py +++ b/test/networkxx_pg_test.py @@ -329,5 +329,3 @@ def test_cytoscape_serialize(self): json_object = json.loads(graph_string) assert(json_object["directed"] is False) assert(len(json_object["elements"]["nodes"]) == 17) - - diff --git a/test/slice_topology_test.py b/test/slice_topology_test.py index a1ae2f2..0651afd 100644 --- a/test/slice_topology_test.py +++ b/test/slice_topology_test.py @@ -1140,3 +1140,33 @@ def __init__(self, val): n1.layout_data = None self.assertIsNone(n1.layout_data) + def test_SubInterface_NetworkX(self): + t = f.ExperimentTopology() + n1 = t.add_node(name='Node1', site='RENC') + n1_nic1 = n1.add_component(ctype=f.ComponentType.SmartNIC, model='ConnectX-6', name='nic1') + n1_nic1_interface1 = n1_nic1.interface_list[0] + + from fim.user.model_element import TopologyException + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child1") + + ch1 = n1_nic1_interface1.add_child_interface(name="child1", labels=f.Labels(vlan="100")) + + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child1", labels=f.Labels(vlan="200")) + + with self.assertRaises(TopologyException): + n1_nic1_interface1.add_child_interface(name="child2", labels=f.Labels(vlan="100")) + + ch2 = n1_nic1_interface1.add_child_interface(name="child2", labels=f.Labels(vlan="200")) + + t.add_network_service(name="net1", nstype=f.ServiceType.L2Bridge, interfaces=[ch1, ch2]) + + t.validate() + t.network_services["net1"].disconnect_interface(ch1) + n1_nic1_interface1.remove_child_interface(name="child1") + t.validate() + t.remove_network_service("net1") + t.validate() + n1_nic1_interface1.remove_child_interface(name="child2") + t.validate() \ No newline at end of file diff --git a/test/test_load.py b/test/test_load.py index 5659fcd..c21b689 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -21,4 +21,4 @@ def testNetworkXLoad(self): print(f"{t1.nodes['n2'].management_ip=}") t2 = load(file_name="test/after.graphml") print(f"{t2.nodes['n2'].management_ip=}") - self.assertEqual(str(t2.nodes['n2'].management_ip), '128.163.179.51', "No management IP address found") \ No newline at end of file + self.assertEqual(str(t2.nodes['n2'].management_ip), '128.163.179.51', "No management IP address found") diff --git a/test/zz_neo4j_pg_test.py b/test/zz_neo4j_pg_test.py index 8daae2b..b58f704 100644 --- a/test/zz_neo4j_pg_test.py +++ b/test/zz_neo4j_pg_test.py @@ -22,7 +22,7 @@ class Neo4jTests(unittest.TestCase): neo4j = {"url": "neo4j://0.0.0.0:7687", "user": "neo4j", "pass": "password", - "import_host_dir": "neo4j/imports/", + "import_host_dir": "neo4j/imports", "import_dir": "/imports"} FIM_CONFIG_YAML = "./fim_config.yml"