Explainability with hcs

docs/SchemeFileFormat.md

@@ -86,7 +86,7 @@ The supported entries in the outputConfiguration object are as follows:
 | subset           | A dict object with the defined subset elements to display in the output                                         | [subset](#subset) object                                     |
 | fullExplanation  | Choose if to print all counterexamples causing the query result in the output                                   | bool                                                         |
 | excludeIPv6Range | If the policies of the config do not contain any IPv6 addresses, do not include IPv6 range in the query results | bool  [default: True]                                        |
-
+| explain          |A pair of node names to explain the policies affecting their connection or lack of it. When 'ALL' is supplied, all the pairs in the topology are explained.| string [ ns/node1,ns/node2 / ALL ]                           |
 
 #### <a name="subset"></a>Subset object
 The supported entries in the subset object are as follows:

nca/FWRules/ConnectivityGraph.py

@@ -343,7 +343,7 @@ def get_connectivity_dot_format_str(self, connectivity_restriction=None):
 
         dot_graph = DotGraph(name)
         peers_groups = self._get_equals_groups()
-        # we are going to treat a a peers_group as one peer.
+        # we are going to treat a peers_group as one peer.
         # the first peer in the peers_group is representing the group
         # we will add the text of all the peers in the group to this peer
         for peers_group, group_connection in peers_groups:

nca/FWRules/FWRule.py

@@ -383,7 +383,8 @@ def get_pod_str(self):
         """
         :return: string for the field src_pods or dst_pods in representation for txt rule format
         """
-        return f'[{self._get_pods_names()}]'
+        sorted_pods_names = ', '.join(sorted(self._get_pods_names().split(', ')))
+        return f'[{sorted_pods_names}]'
 
     def _get_pods_names(self):
         res = ''

nca/FWRules/InteractiveConnectivityGraph.py

@@ -127,7 +127,7 @@ def read_input_file(self):
             """
             try:
                 with open(self.input_svg_file) as svg_file:
-                    self.soup = BeautifulSoup(svg_file.read(), 'xml')
+                    self.soup = BeautifulSoup(svg_file.read(), 'html')
             except Exception as e:
                 print(f'Failed to open file: {self.input_svg_file}\n{e} for reading', file=sys.stderr)
 

nca/FileScanners/GenericTreeScanner.py

@@ -21,6 +21,7 @@ class YamlFile:
 
 class ObjectWithLocation:
     line_number = 0
+    path = ''
     column_number = 0
 
 
@@ -36,13 +37,15 @@ def to_yaml_objects(yaml_node):
     if isinstance(yaml_node, yaml.SequenceNode):
         res = YamlList()
         res.line_number = yaml_node.start_mark.line
+        res.path = yaml_node.start_mark.name
         res.column_number = yaml_node.start_mark.column
         for obj in yaml_node.value:
             res.append(to_yaml_objects(obj))
         return res
     if isinstance(yaml_node, yaml.MappingNode):
         res = YamlDict()
         res.line_number = yaml_node.start_mark.line + 1
+        res.path = yaml_node.start_mark.name
         res.column_number = yaml_node.start_mark.column + 1
         for obj in yaml_node.value:
             res[obj[0].value] = to_yaml_objects(obj[1])

nca/NetworkConfig/NetworkConfig.py

@@ -9,6 +9,7 @@
 from nca.CoreDS.ConnectivityProperties import ConnectivityProperties
 from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections
 from .NetworkLayer import NetworkLayersContainer, NetworkLayerName
+from nca.Utils.ExplTracker import ExplTracker
 
 
 @dataclass
@@ -275,6 +276,8 @@ def allowed_connections_optimized(self, layer_name=None):
         :return: allowed_conns: all allowed connections for relevant peers.
         :rtype: OptimizedPolicyConnections
         """
+        if ExplTracker().is_active():
+            ExplTracker().set_peers(self.peer_container.peer_set)
         if layer_name is not None:
             if layer_name not in self.policies_container.layers:
                 return self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container,

nca/NetworkConfig/NetworkConfigQuery.py

@@ -24,6 +24,7 @@
     PoliciesAndRulesExplanations, PodsListsExplanations, ConnectionsDiffExplanation, IntersectPodsExplanation, \
     PoliciesWithCommonPods, PeersAndConnections, ComputedExplanation
 from .NetworkLayer import NetworkLayerName
+from nca.Utils.ExplTracker import ExplTracker
 
 
 class QueryType(Enum):
@@ -833,11 +834,15 @@ def compute_connectivity_output_optimized(self):
                            ConnectivityProperties.make_conn_props_from_dict({"dst_peers": subset_peers})
             all_conns_opt &= subset_conns
         all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare)
+        expl_conns = all_conns_opt
         if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio):
             output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \
                 self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare)
+            expl_conns, _ = self.convert_props_to_split_by_tcp(all_conns_opt)
         else:
             output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare)
+        if ExplTracker().is_active():
+            ExplTracker().set_connections_and_peers(expl_conns, subset_peers)
         return output_res, opt_fw_rules, opt_fw_rules_tcp, opt_fw_rules_non_tcp
 
     def exec(self):
@@ -922,7 +927,9 @@ def get_props_output_full(self, props, peers_to_compare):
         if self.output_config.outputFormat in ['dot', 'jpg']:
             dot_full = self.dot_format_from_props(props, peers_to_compare)
             return dot_full, None
-        # TODO - handle 'txt_no_fw_rules' output format
+        if self.output_config.outputFormat == 'txt_no_fw_rules':
+            conns_wo_fw_rules = self.txt_no_fw_rules_format_from_props(props, peers_to_compare)
+            return conns_wo_fw_rules, None
         # handle other formats
         formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare)
         return formatted_rules, fw_rules
@@ -988,7 +995,12 @@ def get_props_output_split_by_tcp(self, props, peers_to_compare):
             # concatenate the two graphs into one dot file
             res_str = dot_tcp + dot_non_tcp
             return res_str, None, None
-        # TODO - handle 'txt_no_fw_rules' output format
+        if self.output_config.outputFormat in ['txt_no_fw_rules']:
+            txt_no_fw_rules_tcp = self.txt_no_fw_rules_format_from_props(props_tcp, peers_to_compare, connectivity_tcp_str)
+            txt_no_fw_rules_non_tcp = self.txt_no_fw_rules_format_from_props(props_non_tcp, peers_to_compare,
+                                                                             connectivity_non_tcp_str)
+            res_str = txt_no_fw_rules_tcp + txt_no_fw_rules_non_tcp
+            return res_str, None, None
         # handle formats other than dot and txt_no_fw_rules
         formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, connectivity_tcp_str)
         formatted_rules_non_tcp, fw_rules_non_tcp = self.fw_rules_from_props(props_non_tcp, peers_to_compare,
@@ -1055,6 +1067,20 @@ def dot_format_from_props(self, props, peers, connectivity_restriction=None):
             conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), self.config.peer_container)
         return conn_graph.get_connectivity_dot_format_str(connectivity_restriction)
 
+    def txt_no_fw_rules_format_from_props(self, props, peers, connectivity_restriction=None):
+        """
+        :param ConnectivityProperties props: properties describing allowed connections
+        :param PeerSet peers: the peers to consider for dot output
+        :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to
+               TCP / non-TCP , or not
+        :rtype str
+        :return the connectivity map in txt_no_fw_rules format, considering connectivity_restriction if required
+        """
+        conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config)
+        for cube in props:
+            conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), self.config.peer_container)
+        return conn_graph.get_connections_without_fw_rules_txt_format(connectivity_restriction)
+
     def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None):
         """
         :param dict connections: the connections' dict (map from connection-set to peer pairs)

nca/NetworkConfig/NetworkConfigQueryRunner.py

@@ -11,6 +11,7 @@
 from nca.Resources.NetworkPolicy import NetworkPolicy
 from .NetworkConfig import NetworkConfig
 from . import NetworkConfigQuery
+from nca.Utils.ExplTracker import ExplTracker
 
 
 @dataclass
@@ -40,9 +41,9 @@ def compute_final_results(self, output_format):
         extracts the final query results from self variables
         from self.query_iterations_output computes the final str output of the query,
         other results returned as is from query_result.
-        :param str output_format: the output format to form the final output
-        if output format is json, dumps the output list into one-top-leveled string
-        if output format is yaml, dumps the output list into str of a list of yaml objects
+        :param str output_format: the output format to form the final output.
+        if output format is json, dumps the output list into one-top-leveled string.
+        if output format is yaml, dumps the output list into str of a list of yaml objects.
         otherwise, writes the output list items split by \n
         :return the results: numerical result, output - str , num of not executed
         :rtype: int, str, int
@@ -169,7 +170,12 @@ def _run_query_for_each_config(self):
         query_result = QueryResult()
         for config in self.configs_array:
             query_result.update(self._execute_one_config_query(self.query_name, self._get_config(config)))
-        return query_result.compute_final_results(self.output_configuration.outputFormat)
+        expl_out = ''
+        if ExplTracker().is_active() and self.output_configuration.explain and \
+                ExplTracker().is_output_format_supported(self.output_configuration.outputFormat):
+            expl_out = ExplTracker().explain(self.output_configuration.explain.split(','))
+        numerical_result, output, num_not_executed = query_result.compute_final_results(self.output_configuration.outputFormat)
+        return numerical_result, output + expl_out, num_not_executed
 
     def _run_query_on_configs_vs_base_config(self, cmd_line_flag):
         query_result = QueryResult()

nca/NetworkConfig/NetworkLayer.py

@@ -12,6 +12,7 @@
 from nca.CoreDS.ProtocolSet import ProtocolSet
 from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy
 from nca.Resources.NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy
+from nca.Utils.ExplTracker import ExplTracker
 
 
 # TODO: add a layer for connectivity based on service type (culsterIP / LB / NodePort)? / containers ports?
@@ -252,6 +253,17 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli
         """
         res_conns = OptimizedPolicyConnections()
         for policy in self.policies_list:
+            # Track the peers that were affected by this policy
+            if ExplTracker().is_active():
+                for peer in policy.selected_peers:
+                    src_peers, _ = ExplTracker().extract_peers(policy.optimized_allow_ingress_props)
+                    _, dst_peers = ExplTracker().extract_peers(policy.optimized_allow_egress_props)
+                    peer_name = peer.full_name()
+                    ExplTracker().add_peer_policy(peer_name,
+                                                  policy.name,
+                                                  dst_peers,
+                                                  src_peers,
+                                                  )
             policy_conns = policy.allowed_connections_optimized(is_ingress)
             if policy_conns.captured:  # not empty
                 if captured_func(policy):
@@ -309,6 +321,12 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container):
             else:
                 conn_cube.update({"src_peers": not_captured_not_hep, "dst_peers": all_peers_and_ips})
             not_captured_not_hep_conns = ConnectivityProperties.make_conn_props(conn_cube)
+            if ExplTracker().is_active():
+                src_peers, dst_peers = ExplTracker().extract_peers(not_captured_not_hep_conns)
+                ExplTracker().add_default_policy(src_peers,
+                                                 dst_peers,
+                                                 is_ingress
+                                                 )
             res_conns.all_allowed_conns |= not_captured_not_hep_conns
 
         captured_not_hep = base_peer_set_no_hep & res_conns.captured
@@ -370,12 +388,14 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container):
                                                                                  "dst_peers": non_captured_peers})
                 res_conns.all_allowed_conns |= all_nc_conns - res_conns.denied_conns
                 non_captured_dns_entries = dns_entries - res_conns.captured
+                non_captured_conns = all_nc_conns - res_conns.denied_conns
                 if non_captured_dns_entries:
                     # update allowed non-captured conns to DNSEntry dst with TCP only
                     all_nc_dns_conns = \
                         ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips,
                                                                           "dst_peers": non_captured_dns_entries,
                                                                           "protocols": tcp_protocol})
+                    non_captured_conns |= all_nc_dns_conns
                     res_conns.all_allowed_conns |= all_nc_dns_conns
             else:
                 nc_all_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers,
@@ -385,7 +405,15 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container):
                 nc_dns_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers,
                                                                                  "dst_peers": dns_entries,
                                                                                  "protocols": tcp_protocol})
+                non_captured_conns = nc_all_conns - res_conns.denied_conns
+                non_captured_conns |= nc_dns_conns
                 res_conns.all_allowed_conns |= nc_dns_conns
+            if ExplTracker().is_active():
+                src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns)
+                ExplTracker().add_default_policy(src_peers,
+                                                 dst_peers,
+                                                 is_ingress
+                                                 )
         return res_conns
 
 
@@ -406,6 +434,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container):
         res_conns = OptimizedPolicyConnections()
         all_peers_and_ips = peer_container.get_all_peers_group(True)
         all_peers_no_ips = peer_container.get_all_peers_group()
+        non_captured_conns = None
         if is_ingress:
             # everything is allowed and non captured
             non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips,
@@ -418,4 +447,10 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container):
                 non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers,
                                                                                        "dst_peers": all_peers_and_ips})
                 res_conns.all_allowed_conns = res_conns.allowed_conns | non_captured_conns
+        if non_captured_conns and ExplTracker().is_active():
+            src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns)
+            ExplTracker().add_default_policy(src_peers,
+                                             dst_peers,
+                                             is_ingress
+                                             )
         return res_conns