Minimize alerts information (#38128)

* first commit * fix pre-commit * after testing * change classifier * remove not needed * additionaldata * additionaldata * additionaldata * additionaldata * additionaldata * fix if for adding additinal_data * add under score- remove legacy checked * fix spaces * fix spaces * fix spaces * rn * added more uniq to incoming mapper * fix rn * add incident fields rn * add incident fields rn * fix * bc json * fix merge conflicts * add _ to file_artifacts and newtwork_artifacts * no default null_values bug * fix rn * fix rn * fix rn * fix tests * fix tests * fix tests * fix tests and pre-commit * fix pre-commit * delete later * revert yml * fix bc * fix handlig the excluded_incident_fields param * Bump pack from version CortexXDR to 6.2.3. * small param fix * fix rn * fix rm * fix code after mid demo * fix rn * fix rn * add playbook fixes for word1_word2 * Apply suggestions from code review docs review Co-authored-by: ShirleyDenkberg <[email protected]> * fix rn * add playbook fixes for word1_word2 * add playbook fixes for word1_word2 * add playbook fixes for word1_word2 * add playbook fixes for word1_word2 * fix tests * fix tests * fix tests * fix pre-commit * fix layout * fix layout * rn * fix pre-commit * fix pre-commit * fix pre-commit * fix pre-commit * Bump pack from version CortexXDR to 6.2.6. * fix pre-commit * fix rn * second commit * Update Packs/CortexXDR/ReleaseNotes/6_2_7.json Co-authored-by: JudithB <[email protected]> * third commit * fix test * fix test * Apply suggestions from code review docs review Co-authored-by: ShirleyDenkberg <[email protected]> * small fixes * small fixes * small fixes * small fixes * Bump pack from version CortexXDR to 6.2.9. * delete comment * fix test * fix conflicts * fix version * fix version * fix rm --------- Co-authored-by: Content Bot <[email protected]> Co-authored-by: ShirleyDenkberg <[email protected]> Co-authored-by: JudithB <[email protected]>
demisto · Jan 30, 2025 · 4ab1d64 · 4ab1d64
1 parent 5efc6e0
commit 4ab1d64
Show file tree

Hide file tree

Showing 7 changed files with 303 additions and 18 deletions.
diff --git a/Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.py b/Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.py
@@ -17,6 +17,7 @@
 XDR_INCIDENT_TYPE_NAME = 'Cortex XDR Incident Schema'
 INTEGRATION_NAME = 'Cortex XDR - IR'
 ALERTS_LIMIT_PER_INCIDENTS: int = -1
+REMOVE_ALERTS_NULL_VALUES = 'null_values'
 FIELDS_TO_EXCLUDE = [
     'network_artifacts',
     'file_artifacts'
@@ -188,6 +189,22 @@ def to_xdr_status(status):
                                          if direction == XSOAR_TO_XDR else xsoar_statuses))
 
 
+def handle_excluded_data_from_alerts_param(excluded_alert_fields: list = []) -> Tuple[list, bool]:
+    """handles the excluded_alert_fields parameter
+
+    Args:
+        excluded_alert_fields (list, optional): the fields from alerts to exclude. Defaults to [].
+
+    Returns:
+        (list, bool): (Which fields of alerts should be excluded from the response,
+        and whether null values should be excluded from the response)
+    """
+    remove_nulls_from_alerts = REMOVE_ALERTS_NULL_VALUES in excluded_alert_fields
+    demisto.debug(f"handle_excluded_data_from_alerts_param {remove_nulls_from_alerts=}, {excluded_alert_fields=}")
+    formatted_excluded_data = [field for field in excluded_alert_fields if field != REMOVE_ALERTS_NULL_VALUES]
+    return formatted_excluded_data, remove_nulls_from_alerts
+
+
 class Client(CoreClient):
     def __init__(self, base_url, proxy, verify, timeout, params=None):
         if not params:
@@ -305,7 +322,10 @@ def update_incident(self, incident_id, status=None, assigned_user_mail=None, ass
             timeout=self.timeout
         )
 
-    def get_incident_extra_data(self, incident_id, alerts_limit=1000):
+    def get_incident_extra_data(self, incident_id, alerts_limit=1000,
+                                exclude_artifacts: bool = False,
+                                excluded_alert_fields: List = [],
+                                remove_nulls_from_alerts: bool = False):
         """
         Returns incident by id
 
@@ -317,6 +337,10 @@ def get_incident_extra_data(self, incident_id, alerts_limit=1000):
             'incident_id': incident_id,
             'alerts_limit': alerts_limit,
         }
+        if excluded_alert_fields:
+            request_data['alert_fields_to_exclude'] = excluded_alert_fields
+        if remove_nulls_from_alerts:
+            request_data['drop_nulls'] = True
 
         reply = self._http_request(
             method='POST',
@@ -327,7 +351,10 @@ def get_incident_extra_data(self, incident_id, alerts_limit=1000):
         )
 
         incident = reply.get('reply')
-
+        # workaround for excluding fields which is not supported with the get_incident_extra_data endpoint
+        if exclude_artifacts:
+            for field in FIELDS_TO_EXCLUDE:
+                incident.pop(field, None)
         return incident
 
     def save_modified_incidents_to_integration_context(self):
@@ -387,7 +414,8 @@ def get_tenant_info(self):
     def get_multiple_incidents_extra_data(self, exclude_artifacts, incident_id_list=[],
                                           gte_creation_time_milliseconds=0, statuses=[],
                                           starred=None, starred_incidents_fetch_window=None,
-                                          page_number=0, limit=100):
+                                          page_number=0, limit=100, excluded_alert_fields=[],
+                                          remove_nulls_from_alerts=False):
         """
         Returns incident by id
         :param incident_id_list: The list ids of incidents
@@ -412,8 +440,13 @@ def get_multiple_incidents_extra_data(self, exclude_artifacts, incident_id_list=
                 'operator': 'in',
                 'value': statuses
             })
+        demisto.debug(f"{excluded_alert_fields=}, {remove_nulls_from_alerts=}, {exclude_artifacts=}")
         if exclude_artifacts:
-            request_data['fields_to_exclude'] = FIELDS_TO_EXCLUDE  # type: ignore
+            request_data['fields_to_exclude'] = FIELDS_TO_EXCLUDE
+        if excluded_alert_fields:
+            request_data['alert_fields_to_exclude'] = excluded_alert_fields
+        if remove_nulls_from_alerts:
+            request_data['drop_nulls'] = True
 
         if starred and starred_incidents_fetch_window:
             filters.append({
@@ -613,6 +646,9 @@ def get_incident_extra_data_command(client, args):
     incident_id = args.get('incident_id')
     alerts_limit = int(args.get('alerts_limit', 1000))
     exclude_artifacts = argToBoolean(args.get('excluding_artifacts', 'False'))
+    alert_fields_to_exclude = args.get('alert_fields_to_exclude', [])
+    drop_nulls = args.get('drop_nulls', False)
+    demisto.debug(f"{exclude_artifacts=} , {alert_fields_to_exclude=}, {drop_nulls=}")
     return_only_updated_incident = argToBoolean(args.get('return_only_updated_incident', 'False'))
     if return_only_updated_incident:
         last_mirrored_in_time = get_last_mirrored_in_time(args)
@@ -623,7 +659,10 @@ def get_incident_extra_data_command(client, args):
 
         else:  # the incident was not modified
             return "The incident was not modified in XDR since the last mirror in.", {}, {}
-    raw_incident = client.get_multiple_incidents_extra_data(incident_id_list=[incident_id], exclude_artifacts=exclude_artifacts)
+    raw_incident = client.get_multiple_incidents_extra_data(incident_id_list=[incident_id],
+                                                            exclude_artifacts=exclude_artifacts,
+                                                            excluded_alert_fields=alert_fields_to_exclude,
+                                                            remove_nulls_from_alerts=drop_nulls)
     if not raw_incident:
         raise DemistoException(f'Incident {incident_id} is not found')
     if isinstance(raw_incident, list):
@@ -632,7 +671,9 @@ def get_incident_extra_data_command(client, args):
         demisto.debug(f'for incident:{incident_id} using the old call since "\
             "alert_count:{raw_incident.get("incident", {}).get("alert_count")} >" \
             "limit:{ALERTS_LIMIT_PER_INCIDENTS}')
-        raw_incident = client.get_incident_extra_data(incident_id, alerts_limit)
+        raw_incident = client.get_incident_extra_data(
+            incident_id, alerts_limit, exclude_artifacts=exclude_artifacts,
+            excluded_alert_fields=alert_fields_to_exclude, remove_nulls_from_alerts=drop_nulls)
     readable_output = [tableToMarkdown(f'Incident {incident_id}', raw_incident.get('incident'), removeNull=True)]
 
     incident = sort_incident_data(raw_incident)
@@ -951,7 +992,8 @@ def get_modified_remote_data_command(client, args, mirroring_last_update: str =
     return GetModifiedRemoteDataResponse(list(id_to_modification_time.keys())), last_run_mirroring_str
 
 
-def get_remote_data_command(client, args):
+def get_remote_data_command(client, args, excluded_alert_fields=[], remove_nulls_from_alerts=False):
+    demisto.debug(f"{excluded_alert_fields=}, {remove_nulls_from_alerts=}")
     remote_args = GetRemoteDataArgs(args)
     demisto.debug(f'Performing get-remote-data command with incident id: {remote_args.remote_incident_id}')
 
@@ -960,11 +1002,15 @@ def get_remote_data_command(client, args):
         # when Demisto version is 6.1.0 and above, this command will only be automatically executed on incidents
         # returned from get_modified_remote_data_command so we want to perform extra-data request on those incidents.
         return_only_updated_incident = not is_demisto_version_ge('6.1.0')  # True if version is below 6.1 else False
-
-        incident_data = get_incident_extra_data_command(client, {"incident_id": remote_args.remote_incident_id,
-                                                                 "alerts_limit": 1000,
-                                                                 "return_only_updated_incident": return_only_updated_incident,
-                                                                 "last_update": remote_args.last_update})
+        requested_data = {"incident_id": remote_args.remote_incident_id,
+                          "alerts_limit": 1000,
+                          "return_only_updated_incident": return_only_updated_incident,
+                          "last_update": remote_args.last_update}
+        if excluded_alert_fields:
+            requested_data['alert_fields_to_exclude'] = excluded_alert_fields
+        if remove_nulls_from_alerts:
+            requested_data['drop_nulls'] = True
+        incident_data = get_incident_extra_data_command(client, requested_data)
         if 'The incident was not modified' not in incident_data[0]:
             demisto.debug(f"Updating XDR incident {remote_args.remote_incident_id}")
 
@@ -1099,7 +1145,7 @@ def update_related_alerts(client: Client, args: dict):
     if not new_status:
         raise DemistoException(f"Failed to update alerts related to incident {incident_id},"
                                "no status found")
-    incident_extra_data = client.get_incident_extra_data(incident_id)
+    incident_extra_data = client.get_incident_extra_data(incident_id=incident_id)
     if 'alerts' in incident_extra_data and 'data' in incident_extra_data['alerts']:
         alerts_array = incident_extra_data['alerts']['data']
         related_alerts_ids_array = [str(alert['alert_id']) for alert in alerts_array if 'alert_id' in alert]
@@ -1110,7 +1156,8 @@ def update_related_alerts(client: Client, args: dict):
 
 def fetch_incidents(client: Client, first_fetch_time, integration_instance, exclude_artifacts: bool,
                     last_run: dict, max_fetch: int = 10, statuses: list = [],
-                    starred: Optional[bool] = None, starred_incidents_fetch_window: str = None):
+                    starred: Optional[bool] = None, starred_incidents_fetch_window: str = None,
+                    excluded_alert_fields: list = [], remove_nulls_from_alerts: bool = True):
     global ALERTS_LIMIT_PER_INCIDENTS
     # Get the last fetch time, if exists
     last_fetch = last_run.get('time')
@@ -1142,7 +1189,8 @@ def fetch_incidents(client: Client, first_fetch_time, integration_instance, excl
             # There might be a case where deduped incident doesn't come back and we are returning more than the limit.
             statuses=statuses, limit=max_fetch + len(dedup_incidents), starred=starred,
             starred_incidents_fetch_window=starred_incidents_fetch_window,
-            exclude_artifacts=exclude_artifacts,
+            exclude_artifacts=exclude_artifacts, excluded_alert_fields=excluded_alert_fields,
+            remove_nulls_from_alerts=remove_nulls_from_alerts
         )
 
     # remove duplicate incidents
@@ -1165,7 +1213,10 @@ def fetch_incidents(client: Client, first_fetch_time, integration_instance, excl
             if alert_count > ALERTS_LIMIT_PER_INCIDENTS:
                 demisto.debug(f'for incident:{incident_id} using the old call since alert_count:{alert_count} >" \
                               "limit:{ALERTS_LIMIT_PER_INCIDENTS}')
-                raw_incident_ = client.get_incident_extra_data(incident_id=incident_id)
+                raw_incident_ = client.get_incident_extra_data(incident_id=incident_id,
+                                                               exclude_artifacts=exclude_artifacts,
+                                                               excluded_alert_fields=excluded_alert_fields,
+                                                               remove_nulls_from_alerts=remove_nulls_from_alerts)
                 incident_data = sort_incident_data(raw_incident_)
             sort_all_list_incident_fields(incident_data)
             incident_data |= {
@@ -1356,6 +1407,8 @@ def main():  # pragma: no cover
     starred = True if params.get('starred') else None
     starred_incidents_fetch_window = params.get('starred_incidents_fetch_window', '3 days')
     exclude_artifacts = argToBoolean(params.get('exclude_fields', True))
+    excluded_alert_fields = argToList(params.get('excluded_alert_fields'))
+    excluded_alert_fields, remove_nulls_from_alerts = handle_excluded_data_from_alerts_param(excluded_alert_fields)
     xdr_delay = arg_to_number(params.get('xdr_delay')) or 1
     try:
         timeout = int(params.get('timeout', 120))
@@ -1397,6 +1450,8 @@ def main():  # pragma: no cover
                                                   statuses=statuses,
                                                   starred=starred,
                                                   starred_incidents_fetch_window=starred_incidents_fetch_window,
+                                                  excluded_alert_fields=excluded_alert_fields,
+                                                  remove_nulls_from_alerts=remove_nulls_from_alerts
                                                   )
             demisto.debug(f"Finished a fetch incidents cycle, {next_run=}."
                           f"Fetched {len(incidents)} incidents.")
@@ -1562,7 +1617,9 @@ def main():  # pragma: no cover
             return_results(get_mapping_fields_command())
 
         elif command == 'get-remote-data':
-            return_results(get_remote_data_command(client, args))
+            return_results(get_remote_data_command(client, args,
+                                                   excluded_alert_fields,
+                                                   remove_nulls_from_alerts))
 
         elif command == 'update-remote-system':
             return_results(update_remote_system_command(client, args))

diff --git a/Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.yml b/Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.yml
@@ -192,6 +192,14 @@ configuration:
   section: Collect
   defaultvalue: 'true'
   advanced: true
+- additionalinfo: "Whether to fetch only the essential alert fields in order to minimize the incident's information. Possible values: null_values to remove all null values from alerts data, or any other field of an alert."
+  display: Minimize Alert Information
+  name: excluded_alert_fields
+  required: false
+  type: 16
+  section: Collect
+  advanced: true
+  defaultvalue: 'null_values'
 - display: Close all related alerts in XDR
   name: close_alerts_in_xdr
   required: false