Merge pull request #102 from fabric-testbed/rel-1.1

Rel 1.1

fim/__init__.py

@@ -1,2 +1,2 @@
 #
-__VERSION__ = "1.0.7"
+__VERSION__ = "1.1.0"

fim/graph/abc_property_graph.py

@@ -45,8 +45,11 @@
 from fim.slivers.network_node import NodeSliver, CompositeNodeSliver
 from fim.slivers.network_link import NetworkLinkSliver
 from fim.slivers.path_info import PathInfo, ERO
+from fim.slivers.tags import Tags
+from fim.slivers.measurement_data import MeasurementData
 from fim.slivers.network_service import NetworkServiceSliver, NetworkServiceInfo, NSLayer
 from fim.graph.abc_property_graph_constants import ABCPropertyGraphConstants
+from fim.slivers.gateway import Gateway
 
 
 class GraphFormat(Enum):
@@ -91,7 +94,11 @@ class ABCPropertyGraph(ABCPropertyGraphConstants):
         "structural_info": ABCPropertyGraphConstants.PROP_STRUCTURAL_INFO,
         "ero": ABCPropertyGraphConstants.PROP_ERO,
         "path_info": ABCPropertyGraphConstants.PROP_PATH_INFO,
-        "controller_url": ABCPropertyGraphConstants.PROP_CONTROLLER_URL
+        "controller_url": ABCPropertyGraphConstants.PROP_CONTROLLER_URL,
+        "gateway": ABCPropertyGraphConstants.PROP_GATEWAY,
+        "mf_data": ABCPropertyGraphConstants.PROP_MEAS_DATA,
+        "tags": ABCPropertyGraphConstants.PROP_TAGS,
+        "boot_script": ABCPropertyGraphConstants.PROP_BOOT_SCRIPT
     }
 
     @abstractmethod
@@ -498,6 +505,11 @@ def base_sliver_to_graph_properties_dict(sliver: BaseSliver) -> Dict[str, str]:
             prop_dict[ABCPropertyGraph.PROP_NODE_MAP] = json.dumps(sliver.node_map)
         # boolean is always there. use json dumps for simplicity
         prop_dict[ABCPropertyGraph.PROP_STITCH_NODE] = json.dumps(sliver.stitch_node)
+        # this is already a JSON dict
+        if sliver.mf_data is not None:
+            prop_dict[ABCPropertyGraph.PROP_MEAS_DATA] = sliver.mf_data.data
+        if sliver.tags is not None:
+            prop_dict[ABCPropertyGraph.PROP_TAGS] = sliver.tags.to_json()
 
         return prop_dict
 
@@ -522,6 +534,8 @@ def node_sliver_to_graph_properties_dict(sliver: NodeSliver) -> Dict[str, str]:
             prop_dict[ABCPropertyGraph.PROP_SITE] = sliver.site
         if sliver.location is not None:
             prop_dict[ABCPropertyGraph.PROP_LOCATION] = sliver.location.to_json()
+        if sliver.boot_script is not None:
+            prop_dict[ABCPropertyGraph.PROP_BOOT_SCRIPT] = sliver.boot_script
 
         return prop_dict
 
@@ -574,6 +588,8 @@ def network_service_sliver_to_graph_properties_dict(sliver: NetworkServiceSliver
             prop_dict[ABCPropertyGraph.PROP_CONTROLLER_URL] = sliver.controller_url
         if sliver.site is not None:
             prop_dict[ABCPropertyGraph.PROP_SITE] = sliver.site
+        if sliver.gateway is not None:
+            prop_dict[ABCPropertyGraph.PROP_GATEWAY] = sliver.gateway.to_json()
 
         return prop_dict
 
@@ -639,7 +655,8 @@ def sliver_to_dict(sliver) -> Dict[str, Any]:
         elif type(sliver) == InterfaceSliver:
             d = ABCPropertyGraph.interface_sliver_to_graph_properties_dict(sliver)
         else:
-            raise RuntimeError(f'JSON Conversion for type {type(sliver)} is not supported.')
+            raise PropertyGraphQueryException(msg=f'JSON Conversion for type {type(sliver)} is not supported.',
+                                              graph_id=None, node_id=None)
 
         return d
 
@@ -680,7 +697,11 @@ def set_base_sliver_properties_from_graph_properties_dict(sliver: BaseSliver, d:
                               node_map=json.loads(d[ABCPropertyGraph.PROP_NODE_MAP]) if
                               d.get(ABCPropertyGraph.PROP_NODE_MAP, None) is not None else None,
                               stitch_node=json.loads(d[ABCPropertyGraph.PROP_STITCH_NODE]) if
-                              d.get(ABCPropertyGraph.PROP_STITCH_NODE, None) is not None else False
+                              d.get(ABCPropertyGraph.PROP_STITCH_NODE, None) is not None else False,
+                              tags=Tags.from_json(d.get(ABCPropertyGraph.PROP_TAGS, None)),
+                              mf_data=MeasurementData(d[ABCPropertyGraph.PROP_MEAS_DATA])
+                                                      if d.get(ABCPropertyGraph.PROP_MEAS_DATA, None)
+                                                         is not None else None
                               )
 
     @staticmethod
@@ -698,7 +719,8 @@ def node_sliver_from_graph_properties_dict(d: Dict[str, str]) -> NodeSliver:
                          allocation_constraints=d.get(ABCPropertyGraph.PROP_ALLOCATION_CONSTRAINTS, None),
                          service_endpoint=d.get(ABCPropertyGraph.PROP_SERVICE_ENDPOINT, None),
                          site=d.get(ABCPropertyGraphConstants.PROP_SITE, None),
-                         location=Location.from_json(d.get(ABCPropertyGraph.PROP_LOCATION, None))
+                         location=Location.from_json(d.get(ABCPropertyGraph.PROP_LOCATION, None)),
+                         boot_script=d.get(ABCPropertyGraph.PROP_BOOT_SCRIPT, None)
                          )
         return n
 
@@ -737,7 +759,8 @@ def network_service_sliver_from_graph_properties_dict(d: Dict[str, str]) -> Netw
                           ero=ERO.from_json(d.get(ABCPropertyGraph.PROP_ERO, None)),
                           path_info=PathInfo.from_json(d.get(ABCPropertyGraph.PROP_PATH_INFO, None)),
                           controller_url=d.get(ABCPropertyGraph.PROP_CONTROLLER_URL, None),
-                          site=d.get(ABCPropertyGraphConstants.PROP_SITE, None)
+                          site=d.get(ABCPropertyGraphConstants.PROP_SITE, None),
+                          gateway=Gateway.from_json(d.get(ABCPropertyGraphConstants.PROP_GATEWAY, None))
                           )
         return ns
 
@@ -1024,7 +1047,8 @@ def add_network_node_sliver(self, *, sliver: NodeSliver):
 
         if not self.check_node_unique(label=ABCPropertyGraph.CLASS_NetworkNode,
                                       name=sliver.resource_name):
-            raise RuntimeError(f'Node name {sliver.resource_name} must be unique.')
+            raise PropertyGraphQueryException(msg=f'Node name {sliver.resource_name} must be unique.',
+                                              graph_id=self.graph_id, node_id=None)
 
         props = self.node_sliver_to_graph_properties_dict(sliver)
         self.add_node(node_id=sliver.node_id, label=ABCPropertyGraph.CLASS_NetworkNode, props=props)
@@ -1083,7 +1107,8 @@ def add_network_service_sliver(self, *, parent_node_id: str, network_service: Ne
         if parent_node_id is None and not self.check_node_unique(label=ABCPropertyGraph.CLASS_NetworkService,
                                                                  name=network_service.resource_name):
             # slice-wide network services must have unique names
-            raise RuntimeError(f'Network service name {network_service.resource_name} must be unique.')
+            raise PropertyGraphQueryException(msg=f'Network service name {network_service.resource_name} must be unique.',
+                                              graph_id=self.graph_id, node_id=parent_node_id)
 
         props = self.network_service_sliver_to_graph_properties_dict(network_service)
         self.add_node(node_id=network_service.node_id, label=ABCPropertyGraph.CLASS_NetworkService, props=props)
@@ -1097,17 +1122,21 @@ def add_network_service_sliver(self, *, parent_node_id: str, network_service: Ne
 
     def add_interface_sliver(self, *, parent_node_id: str, interface: InterfaceSliver):
         """
-        Add interface to a network service, linking back to parent.
-        :param parent_node_id: network service id
+        Add interface to a network service, linking back to parent (node or network service),
+        if provided.
+        For most interface slivers there should be a parent with the exception
+        of FacilityPorts/StitchPorts.
+
+        :param parent_node_id: network service id or other parent if available
         :param interface: interface sliver description
         :return:
         """
         assert interface.node_id is not None
-        assert parent_node_id is not None
 
         props = self.interface_sliver_to_graph_properties_dict(interface)
         self.add_node(node_id=interface.node_id, label=ABCPropertyGraph.CLASS_ConnectionPoint, props=props)
-        self.add_link(node_a=parent_node_id, rel=ABCPropertyGraph.REL_CONNECTS, node_b=interface.node_id)
+        if parent_node_id is not None:
+            self.add_link(node_a=parent_node_id, rel=ABCPropertyGraph.REL_CONNECTS, node_b=interface.node_id)
 
     def get_all_ns_or_link_connection_points(self, link_id: str) -> List[str]:
         """
@@ -1133,9 +1162,10 @@ def get_all_node_or_component_connection_points(self, parent_node_id: str) -> Li
         assert parent_node_id is not None
         labels, parent_props = self.get_node_properties(node_id=parent_node_id)
         if ABCPropertyGraph.CLASS_NetworkNode not in labels and \
-                ABCPropertyGraph.CLASS_Component not in labels:
+                ABCPropertyGraph.CLASS_Component not in labels and \
+                ABCPropertyGraph.CLASS_CompositeNode not in labels:
             raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=parent_node_id,
-                                              msg="Parent node type is not NetworkNode or Component")
+                                              msg="Parent node type is not NetworkNode, CompositeNode or Component")
         nss_ifs = self.get_first_and_second_neighbor(node_id=parent_node_id, rel1=ABCPropertyGraph.REL_HAS,
                                                      node1_label=ABCPropertyGraph.CLASS_NetworkService,
                                                      rel2=ABCPropertyGraph.REL_CONNECTS,
@@ -1174,6 +1204,25 @@ def get_stitch_nodes(self) -> List[str]:
         :return:
         """
 
+    def find_peer_connection_points(self, *, node_id: str) -> List[str] or None:
+        """
+        Find the ids of the peer connection points to this one (connected over a Link)
+        if they exist
+        :param node_id: id of the interface/connection point
+        :return:
+        """
+        assert node_id is not None
+
+        # find id of connection point over a Link
+        candidates = self.get_first_and_second_neighbor(node_id=node_id, rel1=ABCPropertyGraph.REL_CONNECTS,
+                                                        node1_label=ABCPropertyGraph.CLASS_Link,
+                                                        rel2=ABCPropertyGraph.REL_CONNECTS,
+                                                        node2_label=ABCPropertyGraph.CLASS_ConnectionPoint)
+        if len(candidates) == 0:
+            return None
+
+        return [x[1] for x in candidates]
+
 
 class ABCGraphImporter(ABC):
     @abstractmethod

fim/graph/abc_property_graph_constants.py

@@ -66,13 +66,18 @@ class ABCPropertyGraphConstants(ABC):
     PROP_ERO = "ERO"
     PROP_PATH_INFO = "PathInfo"
     PROP_CONTROLLER_URL = "ControllerURL"
+    PROP_GATEWAY = "Gateway"
+    PROP_TAGS = "Tags"
+    PROP_MEAS_DATA = "MeasurementData"
+    PROP_BOOT_SCRIPT = "BootScript"
     # these properties get validated to be valid JSON objects whenever someone validates the graph
     JSON_PROPERTY_NAMES = [PROP_LABELS, PROP_CAPACITIES, PROP_LABEL_DELEGATIONS,
                            PROP_CAPACITY_DELEGATIONS, PROP_LABEL_ALLOCATIONS,
                            PROP_CAPACITY_ALLOCATIONS, PROP_RESERVATION_INFO, PROP_STRUCTURAL_INFO,
-                           PROP_ERO, PROP_PATH_INFO, PROP_CAPACITY_HINTS]
+                           PROP_ERO, PROP_PATH_INFO, PROP_CAPACITY_HINTS, PROP_GATEWAY,
+                           PROP_TAGS, PROP_MEAS_DATA]
     # these properties cannot be unset
-    NO_UNSET_PROPERTIES = [GRAPH_ID, NODE_ID, PROP_TYPE, PROP_CLASS]
+    NO_UNSET_PROPERTIES = [GRAPH_ID, NODE_ID, PROP_TYPE, PROP_CLASS, PROP_NAME]
 
     CLASS_NetworkNode = 'NetworkNode'
     CLASS_NetworkService = 'NetworkService'

fim/graph/data/graph_validation_rules.json

@@ -12,20 +12,20 @@
     "msg": "All nodes must be of class NetworkNode, CompositeNode, Component, ConnectionPoint, NetworkService or Link"
   },
   {
-    "rule": "MATCH (n:NetworkNode {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"Server\", \"Switch\", \"VM\", \"Container\", \"NAS\"])",
-    "msg": "All NetworkNodes muse be of type Server or Switch"
+    "rule": "MATCH (n:NetworkNode {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"Server\", \"Switch\", \"VM\", \"Container\", \"NAS\", \"Facility\"])",
+    "msg": "All NetworkNodes muse be of type Server, Switch, VM, Container, NAS or Facility"
   },
   {
     "rule": "MATCH (n:Component {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"SmartNIC\", \"GPU\", \"FPGA\", \"NVME\", \"SharedNIC\"])",
     "msg": "All Components must be of type SharedNIC, SmartNIC, GPU, FPGA or NVME"
   },
   {
-    "rule": "MATCH (n:ConnectionPoint {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"AccessPort\", \"TrunkPort\", \"ServicePort\", \"DedicatedPort\", \"SharedPort\", \"vInt\"])",
-    "msg": "All ConnectionPoints must be of type AccessPort, TrunkPort, ServicePort, DedicatedPort, SharedPort or vInt"
+    "rule": "MATCH (n:ConnectionPoint {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"AccessPort\", \"TrunkPort\", \"ServicePort\", \"DedicatedPort\", \"SharedPort\", \"vInt\", \"FacilityPort\", \"StitchPort\"])",
+    "msg": "All ConnectionPoints must be of type AccessPort, TrunkPort, ServicePort, DedicatedPort, SharedPort, vInt, FacilityPort or StitchPort"
   },
   {
-    "rule": "MATCH (n:NetworkService {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [ \"P4\", \"OVS\", \"MPLS\", \"L2Path\", \"L2Bridge\", \"L2PTP\", \"L2STS\", \"FABNetv4\", \"FABNetv6\", \"L3VPN\", \"PortMirror\"])",
-    "msg": "All NetworkServices must be of type P4, OVS, MPLS, L2Path, L2Bridge, L2PTP, L2STS, FABNetv4, FABNetv6, L3VPN or PortMirror"
+    "rule": "MATCH (n:NetworkService {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [ \"P4\", \"OVS\", \"MPLS\", \"VLAN\", \"L2Path\", \"L2Bridge\", \"L2PTP\", \"L2STS\", \"FABNetv4\", \"FABNetv6\", \"L3VPN\", \"PortMirror\"])",
+    "msg": "All NetworkServices must be of type P4, OVS, MPLS, VLAN, L2Path, L2Bridge, L2PTP, L2STS, FABNetv4, FABNetv6, L3VPN or PortMirror"
   },
   {
     "rule": "MATCH (n:Link {GraphID: $graphId}) RETURN ALL(r IN collect(n) WHERE r.Type IN [\"Wave\", \"Patch\", \"L2Path\"])",

fim/graph/neo4j_property_graph.py

@@ -706,6 +706,8 @@ def _import_graph(self, graphml_file: str, graph_id: str) -> None:
 
         assert graphml_file is not None
         assert graph_id is not None
+        # delete this graph if it exists
+        self.delete_graph(graph_id=graph_id)
         try:
             with self.driver.session() as session:
                 self.log.debug(f"Loading graph {graph_id} into Neo4j")

fim/graph/networkx_property_graph.py

@@ -687,10 +687,8 @@ def add_graph(self, graph_id: str, graph: nx.Graph) -> None:
             # check this graph_id isn't already present
             existing_graph_nodes = list(nxq.search_nodes(self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
             if len(existing_graph_nodes) > 0:
-                # graph already present, warn and exit
-                if self.log is not None:
-                    self.log.warn('Attempting to insert a graph with the same GraphID, skipping')
-                return
+                # graph already present, delete it so we can replace
+                self.del_graph(graph_id)
             # relabel incoming graph nodes to integers, then merge
             temp_graph = nx.convert_node_labels_to_integers(graph, first_label=self.start_id)
             # set/overwrite GraphID property on all nodes
@@ -707,10 +705,8 @@ def add_graph_direct(self, graph_id: str, graph: nx.Graph) -> None:
             # check this graph_id isn't already present
             existing_graph_nodes = list(nxq.search_nodes(self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
             if len(existing_graph_nodes) > 0:
-                # graph already present, warn and exit
-                if self.log is not None:
-                    self.log.warn('Attempting to insert a graph with the same GraphID, skipping')
-                return
+                # graph already present, delete so we can replace
+                self.del_graph(graph_id)
             # relabel incoming graph nodes to integers, then merge
             temp_graph = nx.convert_node_labels_to_integers(graph, first_label=self.start_id)
             self.start_id = self.start_id + len(temp_graph.nodes())

fim/graph/resources/abc_bqm.py

@@ -39,18 +39,28 @@ class ABCBQMPropertyGraph(ABCPropertyGraph):
     def __init__(self, *, graph_id: str, importer, logger=None):
         super().__init__(graph_id=graph_id, importer=importer, logger=logger)
 
-    def get_all_composite_node_components(self, parent_node_id: str) -> List[str]:
-        """
-        Return a list of components, children of a prent (presumably composite node)
-        :param parent_node_id:
-        :return:
-        """
+    def __get_node_components(self, parent_node_id: str, claz: str) -> List[str]:
         assert parent_node_id is not None
         # check that parent is a NetworkNode
         labels, parent_props = self.get_node_properties(node_id=parent_node_id)
-        if ABCPropertyGraph.CLASS_CompositeNode not in labels:
+        if claz not in labels:
             raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=parent_node_id,
-                                              msg="Parent node type is not Composite")
+                                              msg=f"Parent node type is not {claz}")
         return self.get_first_neighbor(node_id=parent_node_id, rel=ABCPropertyGraph.REL_HAS,
                                        node_label=ABCPropertyGraph.CLASS_Component)
 
+    def get_all_composite_node_components(self, parent_node_id: str) -> List[str]:
+        """
+        Return a list of components, children of a prent (presumably composite node)
+        :param parent_node_id:
+        :return:
+        """
+        return self.__get_node_components(parent_node_id, ABCPropertyGraph.CLASS_CompositeNode)
+
+    def get_all_network_node_components(self, parent_node_id: str) -> List[str]:
+        """
+        Return a list of components, children of a prent (presumably composite node)
+        :param parent_node_id:
+        :return:
+        """
+        return self.__get_node_components(parent_node_id, ABCPropertyGraph.CLASS_NetworkNode)

fim/graph/resources/neo4j_cbm.py

@@ -119,7 +119,7 @@ def merge_adm(self, *, adm: ABCPropertyGraph) -> None:
 
         # clone ADM with temporary ID - we will be changing its properties
         temp_adm_graph_id = str(uuid.uuid4())
-        self.log.info('CREATED TEMPORARY ADM GRAPH ID ' + temp_adm_graph_id +
+        self.log.debug('CREATED TEMPORARY ADM GRAPH ID ' + temp_adm_graph_id +
                       ' when merging graph ' + adm.graph_id + ' into ' + self.graph_id)
         temp_graph = adm.clone_graph(new_graph_id=temp_adm_graph_id)
         # crude typecasting
@@ -132,7 +132,7 @@ def merge_adm(self, *, adm: ABCPropertyGraph) -> None:
         temp_adm_graph.rewrite_delegations(real_adm_id=adm.graph_id)
 
         # update the structural info adm_graph_ids field to a single member list on all CBM nodes
-        si = StructuralInfo().set_fields(adm_graph_ids=[adm.graph_id])
+        si = StructuralInfo(adm_graph_ids=[adm.graph_id])
         temp_adm_graph.update_nodes_property(prop_name=self.PROP_STRUCTURAL_INFO,
                                              prop_val=si.to_json())