Merge pull request #132 from fabric-testbed/rel-1.3

Rel 1.3

README.md

@@ -1,4 +1,4 @@
-[![Requirements Status](https://requires.io/github/fabric-testbed/InformationModel/requirements.svg?branch=rel-1.2)](https://requires.io/github/fabric-testbed/InformationModel/requirements/?branch=rel-1.2)
+[![Requirements Status](https://requires.io/github/fabric-testbed/InformationModel/requirements.svg?branch=master)](https://requires.io/github/fabric-testbed/InformationModel/requirements/?branch=master)
 
 [![PyPI](https://img.shields.io/pypi/v/fabric-fim?style=plastic)](https://pypi.org/manage/project/fabric-fim/releases/)
 

fim/__init__.py

@@ -1,2 +1,3 @@
 #
-__VERSION__ = "1.2.5"
+__VERSION__ = "1.3.2"
+

fim/graph/abc_property_graph.py

@@ -102,7 +102,8 @@ class ABCPropertyGraph(ABCPropertyGraphConstants):
         "mf_data": ABCPropertyGraphConstants.PROP_MEAS_DATA,
         "tags": ABCPropertyGraphConstants.PROP_TAGS,
         "flags": ABCPropertyGraphConstants.PROP_FLAGS,
-        "boot_script": ABCPropertyGraphConstants.PROP_BOOT_SCRIPT
+        "boot_script": ABCPropertyGraphConstants.PROP_BOOT_SCRIPT,
+        "layout": ABCPropertyGraphConstants.PROP_LAYOUT
     }
 
     @abstractmethod
@@ -947,6 +948,56 @@ def build_deep_component_sliver_from_dict(*, props: Dict[str, Any]) -> Component
             cs.network_service_info = nsi
         return cs
 
+    def build_deep_interface_sliver(self, *, node_id: str) -> InterfaceSliver:
+        """
+        Build a deep interface sliver from graph
+        :param node_id:
+        :return:
+        """
+        clazzes, props = self.get_node_properties(node_id=node_id)
+        if ABCPropertyGraph.CLASS_ConnectionPoint not in clazzes:
+            raise PropertyGraphQueryException(node_id=node_id, graph_id=self.graph_id,
+                                              msg="Node is not of class Interface")
+        # create top-level sliver
+        isl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(props)
+        return isl
+
+    @staticmethod
+    def build_deep_interface_sliver_from_dict(*, props: Dict[str, Any]) -> InterfaceSliver:
+        """
+        Build a deep interface sliver from graph
+        :param props:
+        :return:
+        """
+        # create top-level sliver
+        isl = ABCPropertyGraph.interface_sliver_from_graph_properties_dict(props)
+        return isl
+
+    def build_deep_link_sliver(self, *, node_id: str) -> NetworkLinkSliver:
+        """
+        Build a deep link sliver from graph
+        :param node_id:
+        :return:
+        """
+        clazzes, props = self.get_node_properties(node_id=node_id)
+        if ABCPropertyGraph.CLASS_Link not in clazzes:
+            raise PropertyGraphQueryException(node_id=node_id, graph_id=self.graph_id,
+                                              msg="Node is not of class Link")
+        # create top-level sliver
+        lsl = ABCPropertyGraph.link_sliver_from_graph_properties_dict(props)
+        return lsl
+
+    @staticmethod
+    def build_deep_link_sliver_from_dict(*, props: Dict[str, Any]) -> NetworkLinkSliver:
+        """
+        Build a deep link sliver from graph
+        :param props:
+        :return:
+        """
+        # create top-level sliver
+        lsl = ABCPropertyGraph.link_sliver_from_graph_properties_dict(props)
+        return lsl
+
     def remove_network_node_with_components_nss_cps_and_links(self, node_id: str):
         """
         Remove a network node, all of components and their interfaces, parent interfaces
@@ -1210,9 +1261,9 @@ def get_parent(self, node_id: str, rel: str, parent: str) -> Tuple[str, str]:
         :param parent: parent class
         :return:
         """
-        assert node_id is not None
-        assert rel is not None
-        assert parent is not None
+        #assert node_id is not None
+        #assert rel is not None
+        #assert parent is not None
         parent_ids = self.get_first_neighbor(node_id=node_id, rel=rel,
                                              node_label=parent)
         if len(parent_ids) != 1:
@@ -1228,6 +1279,16 @@ def get_stitch_nodes(self) -> List[str]:
         :return:
         """
 
+    @abstractmethod
+    def get_graph_diff(self, other_graph, label: str):
+        """
+        Return two lists - nodes that are in this graph but NOT in the other graph
+        and node that are in the other graph, but not in this graph, using label
+        as a filter
+        [0] - elements present in self, absent in other (i.e. removed elements)
+        [1] - elements present in other, absent in self (i.e. added elements
+        """
+
     def find_peer_connection_points(self, *, node_id: str) -> List[str] or None:
         """
         Find the ids of the peer connection points to this one (connected over a Link)

fim/graph/abc_property_graph_constants.py

@@ -73,12 +73,13 @@ class ABCPropertyGraphConstants(ABC):
     PROP_FLAGS = "Flags"
     PROP_MEAS_DATA = "MeasurementData"
     PROP_BOOT_SCRIPT = "BootScript"
+    PROP_LAYOUT = "Layout"
     # these properties get validated to be valid JSON objects whenever someone validates the graph
     JSON_PROPERTY_NAMES = [PROP_LABELS, PROP_CAPACITIES, PROP_LABEL_DELEGATIONS,
                            PROP_CAPACITY_DELEGATIONS, PROP_LABEL_ALLOCATIONS,
                            PROP_CAPACITY_ALLOCATIONS, PROP_RESERVATION_INFO, PROP_STRUCTURAL_INFO,
                            PROP_ERO, PROP_PATH_INFO, PROP_CAPACITY_HINTS, PROP_GATEWAY,
-                           PROP_TAGS, PROP_FLAGS, PROP_MEAS_DATA]
+                           PROP_TAGS, PROP_FLAGS, PROP_MEAS_DATA, PROP_LAYOUT]
     # these properties cannot be unset
     NO_UNSET_PROPERTIES = [GRAPH_ID, NODE_ID, PROP_TYPE, PROP_CLASS, PROP_NAME]
 

fim/graph/neo4j_property_graph.py

@@ -602,6 +602,23 @@ def check_node_unique(self, *, label: str, name: str) -> bool:
                 return True
             return False
 
+    def get_graph_diff(self, other_graph, label: str):
+        assert other_graph is not None
+        assert label is not None
+
+        query = f"MATCH(n:{label} {{GraphID: $graphIdA}}) WITH n MATCH(n1:{label} {{GraphID: $graphIdB}}) WITH " \
+                f"collect(DISTINCT n) as A, collect(DISTINCT n1) as B, " \
+                f"collect(DISTINCT n.NodeID) as AN, collect(DISTINCT n1.NodeID) as BN " \
+                f"RETURN [x in A WHERE NOT x.NodeID in BN] as AnotB, [x in B WHERE NOT x.NodeID in AN] as BnotA"
+        with self.driver.session() as session:
+            val = session.run(query, graphIdA=self.graph_id, graphIdB=other_graph.graph_id).single()
+            if val is None:
+                raise PropertyGraphQueryException(graph_id=self.graph_id,
+                                                  node_id=None, msg=f"Unable to diff with graph {other_graph.graph_id}")
+            # A not B means element was removed
+            # B not A means element was added
+            return val.data()['AnotB'], val.data()['BnotA']
+
 
 class Neo4jGraphImporter(ABCGraphImporter):
 
@@ -758,10 +775,10 @@ def import_graph_from_string(self, *, graph_string: str, graph_id: str = None) -
                 # something in APOC prevents loading sometimes on some platforms
                 self.log.debug(f"Trying to load the file {mapped_file_name}")
                 self._import_graph(mapped_file_name, assigned_id)
-                retry = -1
+                retry = -5
             except PropertyGraphImportException:
                 self.log.warning(f"Transient error, unable to load, deleting and reimporting graph {assigned_id}")
-                retry = retry - 1
+                retry -= 1
                 self.delete_graph(graph_id=assigned_id)
                 # sleep and try again
                 time.sleep(1.0)
@@ -796,12 +813,28 @@ def import_graph_from_string_direct(self, *, graph_string: str) -> ABCPropertyGr
         # get graph id
         graph_id = self.get_graph_id(graph_file=host_file_name)
 
-        self._import_graph(mapped_file_name, graph_id)
+        retry = APOC_RETRY_COUNT
+        while retry > 0:
+            try:
+                # something in APOC prevents loading sometimes on some platforms
+                self.log.debug(f"Trying to load the file {mapped_file_name}")
+                self._import_graph(mapped_file_name, graph_id)
+                retry = -5
+            except PropertyGraphImportException:
+                self.log.warning(f"Transient error, unable to load, deleting and reimporting graph {graph_id}")
+                retry -= 1
+                self.delete_graph(graph_id=graph_id)
+                # sleep and try again
+                time.sleep(1.0)
 
         # unlink temp file
         self.log.debug(f"Unlinking temporary file {host_file_name}")
         os.unlink(host_file_name)
 
+        if retry == 0:
+            raise PropertyGraphImportException(graph_id=graph_id,
+                                               msg='Unable to load graph after multiple attempts')
+
         return Neo4jPropertyGraph(graph_id=graph_id, importer=self, logger=self.log)
 
     def import_graph_from_file_direct(self, *, graph_file: str) -> ABCPropertyGraph:

fim/graph/networkx_property_graph.py

@@ -664,6 +664,9 @@ def check_node_unique(self, *, label: str, name: str):
                                             ]}))
         return len(graph_nodes) == 0
 
+    def get_graph_diff(self, other_graph, label: str):
+        raise RuntimeError("Not implementable with this backend.")
+
 
 class NetworkXGraphStorage:
     """

fim/graph/networkx_property_graph_disjoint.py

@@ -54,7 +54,7 @@ def merge_nodes(self, node_id: str, other_graph, merge_properties=None):
         has assumptions about a common store for all graphs.
         Would require to store all graphs in a single NetworkX graph.
         """
-        raise RuntimeError("Not implementable")
+        raise RuntimeError("Not implementable with this backend.")
 
     def graph_exists(self) -> bool:
         """

fim/slivers/attached_components.py

@@ -28,6 +28,7 @@
 
 from .base_sliver import BaseSliver
 from .network_service import NetworkServiceInfo
+from .topology_diff import TopologyDiff
 
 
 @enum.unique
@@ -63,6 +64,9 @@ def type_from_str(ctype: str) -> ComponentType or None:
             if ctype == str(t):
                 return t
 
+    def diff(self, other_sliver) -> TopologyDiff or None:
+        raise RuntimeError('Not implemented')
+
 
 class AttachedComponentsInfo:
     """

fim/slivers/base_sliver.py

@@ -33,6 +33,7 @@
 from fim.slivers.delegations import Delegations
 from fim.slivers.tags import Tags
 from fim.slivers.measurement_data import MeasurementData
+from fim.slivers.topology_diff import TopologyDiff, TopologyDiffTuple
 
 
 class BaseSliver(ABC):
@@ -63,6 +64,7 @@ def __init__(self):
         self.flags = None # various flags
         self.mf_data = None # opaque JSON object limited in length
         self.boot_script = None # string limited in length
+        self.layout = None # opaque JSON object limited in length
 
     def set_type(self, resource_type):
         self.resource_type = resource_type
@@ -212,6 +214,26 @@ def set_properties(self, **kwargs):
             # we can set anything the sliver model has a setter for
             self.__getattribute__('set_' + k)(v)
 
+    @staticmethod
+    def _dict_diff(dict_a, dict_b, show_value_diff=False):
+        """
+        supports diffing slivers
+        """
+        result = {'added': {k: dict_b[k] for k in set(dict_b) - set(dict_a)},
+                  'removed': {k: dict_a[k] for k in set(dict_a) - set(dict_b)}}
+        if show_value_diff:
+            common_keys = set(dict_a) & set(dict_b)
+            result['value_diffs'] = {
+                k: (dict_a[k], dict_b[k])
+                for k in common_keys
+                if dict_a[k] != dict_b[k]
+            }
+        return result
+
+    @abstractmethod
+    def diff(self, other_sliver) -> TopologyDiff or None:
+        assert isinstance(self, other_sliver.__class__)
+
     @classmethod
     def list_properties(cls) -> Tuple[str]:
         """
@@ -243,6 +265,17 @@ def get_property(self, prop_name: str):
         """
         return self.__getattribute__('get_' + prop_name)()
 
+    def property_exists(self, prop_name: str):
+        """
+        Does this property have a getter?
+        """
+        try:
+            self.__getattribute__('get_' + prop_name)
+            exists = True
+        except AttributeError:
+            exists = False
+        return exists
+
     def __repr__(self):
         exclude_set = {"get_property", "get_stitch_node"}
         print_set = list()

fim/slivers/capacities_labels.py

@@ -355,7 +355,7 @@ class Labels(JSONField):
                                          0 < int(v.split('-')[1]) <= 4096 and
                                          int(v.split('-')[0]) < int(v.split('-')[1]) else False),
                        "1-4096"),
-        'asn': ((lambda a: True if 0 < int(a) < 65536 else False), "1-65535")
+        'asn': ((lambda a: True if 0 < int(a) < 2**32 else False), "1-4294967295")
     }
 
     def __init__(self, **kwargs):
@@ -561,7 +561,8 @@ class Flags(JSONField):
     JSON-ified representation of various flags that can be attached to slivers
     """
     def __init__(self, **kwargs):
-        self.auto_config = False
+        self.auto_config = False # primarily for interfaces
+        self.auto_mount = False # primarily for storage components
         self._set_fields(**kwargs)
 
     def _set_fields(self, **kwargs):