Merge branch 'dev' into feature/fix_for_berlin_quickfix_multiple_subg…

…rids

dev_requirements.txt

@@ -4,4 +4,5 @@ pytest-parallel
 sphinx
 sphinx_rtd_theme
 Cython
-pymetis
+pymetis
+pyvis

ding0/core/__init__.py

@@ -45,6 +45,7 @@
 from geoalchemy2.shape import from_shape
 import subprocess
 import json
+from pyvis.network import Network as pynet
 
 if not 'READTHEDOCS' in os.environ:
     from shapely.wkt import loads as wkt_loads
@@ -67,7 +68,7 @@
     build_graph_from_ways, create_buffer_polygons, graph_nodes_outside_buffer_polys, \
     compose_graph, get_fully_conn_graph, split_conn_graph, get_outer_conn_graph, \
     handle_detour_edges, add_edge_geometry_entry, remove_unloaded_deadends, \
-    flatten_graph_components_to_lines, subdivide_graph_edges, simplify_graph_adv, \
+    consolidate_edges, subdivide_graph_edges, simplify_graph_adv, \
     create_simple_synthetic_graph
 
 from ding0.grid.lv_grid.clustering import get_cluster_numbers, distance_restricted_clustering
@@ -389,6 +390,52 @@ def run_ding0(
         logger.info("STEP 9: Open all switch disconnectors in MV grid")
         self.control_circuit_breakers(mode='open')
 
+        # ==============================================
+        # Quickfix for https://github.com/openego/ding0/issues/402
+        # Removes all nodes belonging to subgraphs
+        for grid_district in self.mv_grid_districts():
+            subgraphs = [s for s in nx.connected_components(grid_district.mv_grid.graph) if len(s) > 1]
+            subgraph_max_len = 0
+            for s in subgraphs:
+                if len(s) > subgraph_max_len:
+                    subgraph_max_len = len(s)
+
+            # Do we have subgraphs?
+            if len(subgraphs) > 1:
+                # Log
+                errstr = ", ".join([f"Grid {_ + 1} ({len(s)} nodes)" for _, s in enumerate(subgraphs)])
+                logger.error(
+                    f"Grid in {str(grid_district)} has {len(subgraphs)} subgrids (cf. issue #402): {errstr}. "
+                    f"Only the largest graph will be retained."
+                )
+
+                # Plot with pyvis for debugging
+                net = pynet(
+                    directed=False,
+                    select_menu=True,
+                    filter_menu=True,
+                )
+                net.show_buttons()
+                graph_relabeled = grid_district.mv_grid.graph.copy()
+                graph_relabeled = nx.relabel_nodes(
+                    graph_relabeled,
+                    dict(zip(
+                        graph_relabeled.nodes(),
+                        [str(n) for n in graph_relabeled.nodes()]
+                    ))
+                )
+                for n1, n2, d in graph_relabeled.edges(data=True):
+                    d.pop("branch", None)
+                net.from_nx(graph_relabeled)
+                net.write_html(f"{str(grid_district)}_graph_debug.html", notebook=False)
+
+                # Remove all nodes belonging to isolated subgraphs
+                for component in subgraphs:
+                    if len(component) < subgraph_max_len:
+                        for node in component:
+                            grid_district.mv_grid.graph.remove_node(node)
+        # ==============================================
+
         logger.info("STEP 10: Do power flow analysis of MV grid")
         self.run_powerflow(session, method='onthefly', export_pypsa=False, debug=debug)
         if export_mv_figures:
@@ -803,7 +850,7 @@ def import_lv_load_areas_and_build_new_lv_districts(
             # Process outer graph
             outer_graph = get_outer_conn_graph(outer_graph, inner_node_list)
             outer_graph = add_edge_geometry_entry(outer_graph)
-            outer_graph = flatten_graph_components_to_lines(outer_graph, inner_node_list)
+            outer_graph = consolidate_edges(outer_graph, inner_node_list)
             outer_graph = handle_detour_edges(outer_graph, level="mv", mode='remove')
 
             # Compose graph

ding0/grid/lv_grid/graph_processing.py

@@ -520,49 +520,47 @@ def simplify_graph_adv(G, street_load_nodes, strict=True, remove_rings=True):
     logger.debug(msg)
     return G
 
-def flatten_graph_components_to_lines(G, inner_node_list):
+def consolidate_edges(graph, inner_node_list):
     """
     Build single edges based on outter graph component to connect
     isolated components/ nodes in inner graph/ not buffered area.
     """
     # TODO: add edge tags 'highway' and 'osmid' to shortest path edge
 
-    components = list(nx.weakly_connected_components(G))
-    sp_path = lambda p1, p2: nx.shortest_path(G, p1, p2, weight='length') if nx.has_path(G, p1, p2) else None
+    conn_comps = list(nx.weakly_connected_components(graph))
+    sp_path = lambda p1, p2: nx.shortest_path(graph, p1, p2, weight='length') \
+        if nx.has_path(graph, p1, p2) else None
 
     nodes_to_remove = []
     edges_to_add = []
-    common_nodes = set(G.nodes()) & set(inner_node_list)
+    common_nodes = set(graph.nodes()) & set(inner_node_list)
 
-    for comp in components:
-        conn_nodes = list(comp & common_nodes)
-        if len(comp) > 2 and len(conn_nodes) == 1:
-            G.remove_nodes_from(comp) # removes unwanted islands / loops
+    for cc in conn_comps:
+        conn_nodes = list(cc & common_nodes)
+        if len(cc) > 2 and len(conn_nodes) == 1:
+            graph.remove_nodes_from(cc) # removes unwanted islands / loops
 
         else: 
             endpoints = combinations(conn_nodes, 2)
             paths = [sp_path(n[0], n[1]) for n in endpoints]
             for path in paths:
                 geoms = []
                 for u, v in zip(path[:-1], path[1:]):
-                    geom = G.edges[u,v,0]['geometry']
-                    # deprecated due to add_edge_geometry_entry(G)
-                    # try: geom = G.edges[u,v,0]['geometry']
-                    # except: geom = LineString([Point((G.nodes[node]["x"], G.nodes[node]["y"])) for node in [u,v]])
+                    geom = graph.edges[u,v,0]['geometry']
                     geoms.append(geom)
 
                 merged_line = linemerge(MultiLineString(geoms))
                 edges_to_add.append([path[0], path[-1], merged_line])
-                nodes_to_remove.append(list(set(comp) - set(conn_nodes)))
+                nodes_to_remove.append(list(set(cc) - set(conn_nodes)))
 
     for nodes in nodes_to_remove:
-        G.remove_nodes_from(nodes)
+        graph.remove_nodes_from(nodes)
 
     for edge in edges_to_add:
-        G.add_edge(edge[0], edge[1], 0, geometry=edge[2], length=edge[2].length)
-        G.add_edge(edge[1], edge[0], 0, geometry=edge[2], length=edge[2].length)
+        graph.add_edge(edge[0], edge[1], 0, geometry=edge[2], length=edge[2].length)
+        graph.add_edge(edge[1], edge[0], 0, geometry=edge[2], length=edge[2].length)
 
-    return G
+    return graph
 
 def handle_detour_edges(graph, level="mv", mode='remove'):
     """

ding0/grid/mv_grid/urban_mv_connect.py

@@ -41,6 +41,20 @@ def mv_urban_connect(mv_grid, osm_graph_red, core_graph, stub_graph, stub_dict,
     routed_graph_node_set = routed_graph_node_set - forbidden_object_set
     root_nodes_to_remove = []
 
+    # ==============================================
+    # Detect problem with multiple subgraphs, see https://github.com/openego/ding0/issues/402
+    from collections import Counter
+    stub_root_nodes = [s["root"] for s in stub_dict.values()]
+    if len(stub_root_nodes) > len(set(stub_root_nodes)):
+        duplicated_root_nodes = {r: c for r, c in Counter(stub_root_nodes).items() if c > 1}
+        for r, c in duplicated_root_nodes.items():
+            stubs = [s["comp"] for s in stub_dict.values() if s["root"] == r]
+            logger.error(
+                f"Multiple stubs share the same root node, this is likely to result in multiple graphs "
+                f"(cf. issue #402): Root: {r}, Count: {c}, Stubs: {stubs}"
+            )
+    # ==============================================
+
     for key, stub_data in stub_dict.items():
 
         # get root_node, loads (mv_station or mv_load) and cable distributor nodes in osm stub graph