Fix missing updated alerts (demisto#32099) (demisto#32198)

* Add ids to storage (#111)

* Fix create modified alerts (#100)

* Add fetch for uncreated modified alerts

* Fix python format issues

* Add release notes

* Change Last modified fetch string at fetch incidents action (#101)



* Change fetch of alerts timestamp to consider when no alerts are fetched (#104)



* Change fetch of alerts timestamp to consider when no alerts are fetched (#103)



* Fix lookup of the latest timestamp of create/update of an alerts (#106)

* Fix lookup of the latest timestamp of create/update of an alerts

* Fix linter issues

* Add a test to check next_run changes with modified alerts

* Add field to fetch alerts that has been updated

* Update docker image version

* Fix create modified alerts (#100)

* Add fetch for uncreated modified alerts

* Fix python format issues

* Add release notes

* Change fetch of alerts timestamp to consider when no alerts are fetched (#103)



* Fix lookup of the latest timestamp of create/update of an alerts (#106)

* Fix lookup of the latest timestamp of create/update of an alerts

* Fix linter issues

* Add a test to check next_run changes with modified alerts

* Fix missing alerts (#110)

* Add field to fetch alerts that has been updated

* Update docker image version

* Update merge issues

* Update docker image version and release notes

* Fix merge issue

* Update merge issue

---------




* Update lint issues (#112)

* Update ruff issues (#113)

* Change dates to strings when fetching alerts (#116)

* Add source header (#117)

* Add source header in requests

* Update release notes

---------

Co-authored-by: Felipe Garrido <[email protected]>
Co-authored-by: Diego Ramirez R <[email protected]>
Co-authored-by: Diego Ramirez <[email protected]>

Packs/ZeroFox/Integrations/ZeroFox/ZeroFox.py

@@ -4,7 +4,8 @@
 
 """ IMPORTS  """
 from dateparser import parse as parse_date
-from typing import Any
+from datetime import datetime
+from typing import Any, TypedDict
 from collections.abc import Callable
 from requests import Response
 from copy import deepcopy
@@ -14,6 +15,18 @@
 """ GLOBALS / PARAMS  """
 FETCH_TIME_DEFAULT = "3 days"
 CLOSED_ALERT_STATUS = ["Closed", "Deleted"]
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
+MAX_ALERT_IDS_STORED = 200
+
+""" Types """
+FetchIncidentsStorage = TypedDict("FetchIncidentsStorage", {
+    "last_fetched": str,
+    "last_offset": str,
+    "first_run_at": str,
+    "last_modified_fetched": str,
+    "last_modified_offset": str,
+    "zf-ids": list[int],
+})
 
 
 """ CLIENT """
@@ -30,6 +43,7 @@ def __init__(
         }
         self.fetch_limit = fetch_limit
         self.only_escalated = only_escalated
+        self.auth_token = ""
 
     def api_request(
         self,
@@ -101,6 +115,8 @@ def get_authorization_token(self) -> str:
         """
         :return: Returns the authorization token
         """
+        if self.auth_token:
+            return self.auth_token
         url_suffix: str = "/1.0/api-token-auth/"
         response_content = self.api_request(
             "POST",
@@ -110,8 +126,8 @@ def get_authorization_token(self) -> str:
             headers_builder_type=None,
             prefix=None,
         )
-        token = response_content.get("token", "")
-        return token
+        self.auth_token = response_content.get("token", "")
+        return self.auth_token
 
     def _get_new_access_token(self) -> str:
         url_suffix: str = "/auth/token/"
@@ -139,6 +155,7 @@ def get_api_request_header(self) -> dict[str, str]:
             "Authorization": f"Token {token}",
             "Content-Type": "application/json",
             "Accept": "application/json",
+            "zf-source": "XSOAR",
         }
 
     def get_cti_request_header(self) -> dict[str, str]:
@@ -147,6 +164,7 @@ def get_cti_request_header(self) -> dict[str, str]:
             "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
             "Accept": "application/json",
+            "zf-source": "XSOAR",
         }
 
     def get_policy_types(self) -> dict[str, Any]:
@@ -535,6 +553,7 @@ def alert_to_incident(alert: dict[str, Any]) -> dict[str, str]:
     incident = {
         "rawJSON": json.dumps(alert),
         "name": f"ZeroFox Alert {alert_id}",
+        "dbotMirrorId": str(alert_id),
         "occurred": alert.get("timestamp", ""),
     }
     return incident
@@ -1017,86 +1036,181 @@ def get_exploits_content(
     return exploits_content
 
 
-""" COMMANDS """
-
-
-def test_module(client: ZFClient) -> str:
-    """
-    Performs basic get request to get item samples
-    """
-    client.get_policy_types()
-    return "ok"
-
-
-def fetch_incidents(
+def get_incidents_data(
     client: ZFClient,
-    last_run: dict[str, str],
-    first_fetch_time: str
-) -> tuple[dict[str, str], list[dict[str, Any]]]:
-    date_format = "%Y-%m-%dT%H:%M:%S.%f"
-    last_fetched = last_run.get("last_fetched")
-    last_offset_str: str = last_run.get("last_offset", "")
-    if last_fetched is None:
-        last_fetched = first_fetch_time
-    last_fetched = parse_date(last_fetched, date_formats=(date_format,))
-    last_offset = int(last_offset_str) if last_offset_str else 0
-    if last_fetched is None:
-        raise ValueError("last_fetched param is invalid")
+    params: dict[str, Any],
+    is_valid_alert: Callable[[dict[str, Any]], bool] | None = None,
+    timestamp_field: str = "timestamp"
+) -> tuple[list[dict[str, Any]], str, str | None, list[int]]:
+    incidents: list[dict[str, Any]] = []
+    next_offset = "0"
 
-    response_content = client.list_alerts(
-        {
-            "sort_direction": "asc",
-            "min_timestamp": last_fetched,
-            "offset": last_offset,
-        }
-    )
+    response_content = client.list_alerts(params)
     alerts: list[dict[str, Any]] = response_content.get("alerts", [])
 
-    next_run = {
-        "last_fetched": last_fetched.strftime(date_format),
-        "last_offset": str(last_offset),
-    }
-    incidents: list[dict[str, Any]] = []
-
     if not alerts:
-        return next_run, incidents
+        return incidents, next_offset, None, []
 
     integration_instance = demisto.integrationInstance()
+    processed_alerts: list[dict[str, Any]] = []
     for alert in alerts:
+        if is_valid_alert and not is_valid_alert(alert):
+            continue
         # Fields for mirroring alert
         alert["mirror_direction"] = "In"
         alert["mirror_instance"] = integration_instance
 
+        processed_alerts.append(alert)
         incident = alert_to_incident(alert)
         incidents.append(incident)
 
     next_page: str = response_content.get("next", "")
     if next_page:
         parsed_next_page = urlparse.urlparse(next_page)
         parsed_query = urlparse.parse_qs(parsed_next_page.query)
-        next_run["last_offset"] = parsed_query.get("offset", ["0"])[0]
-        return next_run, incidents
+        next_offset = parsed_query.get("offset", ["0"])[0]
 
-    # max_update_time is the timestamp of the last alert in alerts
-    # (alerts is a sorted list by timestamp)
-    last_alert_timestamp = alerts[-1].get("timestamp", "")
+    # last_alert_timestamp is the oldest timestamp in alerts
+    parsed_last_alert: str = params.get('min_timestamp') or params.get('last_modified_min_date') or ""
+    parsed_last_alert_timestamp = parse_date(
+        parsed_last_alert,
+        date_formats=(DATE_FORMAT,),
+    )
+    if parsed_last_alert_timestamp is None:
+        raise ValueError("Incorrect timestamp in params of fetch-incidents")
+    for alert in processed_alerts:
+        alert_timestamp_str: str = alert.get(timestamp_field, "")
+        alert_timestamp = parse_date(
+            alert_timestamp_str,
+            date_formats=(DATE_FORMAT,),
+        )
+        if alert_timestamp is None:
+            raise ValueError("Incorrect timestamp in alert of fetch-incidents")
+        alert_timestamp = alert_timestamp.replace(tzinfo=None)
+        if alert_timestamp > parsed_last_alert_timestamp:
+            parsed_last_alert_timestamp = alert_timestamp
 
     # add 1 millisecond to last alert timestamp,
     # in order to prevent duplicated alerts
-    parsed_last_alert_timestamp = parse_date(
-        last_alert_timestamp,
-        date_formats=(date_format,),
-    )
     if parsed_last_alert_timestamp is None:
-        raise ValueError("Incorrect timestamp in last alert "
-                         "of fetch-incidents")
+        raise ValueError("Incorrect timestamp in last alert of fetch-incidents")
     max_update_time = (
         parsed_last_alert_timestamp + timedelta(milliseconds=1)
-    ).strftime(date_format)
-    next_run["last_fetched"] = max_update_time
-    next_run["last_offset"] = "0"
+    ).strftime(DATE_FORMAT)
+
+    def get_alert_ids(alert: dict[str, Any]) -> int:
+        return alert.get("id") or 0
+    processed_alerts_ids: list[int] = list(map(get_alert_ids, processed_alerts))
+
+    return incidents, next_offset, max_update_time, processed_alerts_ids
+
+
+def parse_last_fetched_date(
+    last_fetched_str: str | None,
+    first_fetch_time: str
+) -> datetime:
+    # If no last_fetched present, use default value
+    if not last_fetched_str:
+        last_fetched_str = first_fetch_time
+    last_fetched = parse_date(last_fetched_str, date_formats=(DATE_FORMAT,))
+    # If last_fetched is invalid, raise ValueError
+    if last_fetched is None:
+        raise ValueError("last_fetched param is invalid")
+    return last_fetched
+
+
+""" COMMANDS """
+
+
+def test_module(client: ZFClient) -> str:
+    """
+    Performs basic get request to get item samples
+    """
+    client.get_policy_types()
+    return "ok"
+
+
+def fetch_incidents(
+    client: ZFClient,
+    last_run: FetchIncidentsStorage,
+    first_fetch_time: str
+) -> tuple[FetchIncidentsStorage, list[dict[str, Any]]]:
+    # Last fetched date
+    last_fetched_str = last_run.get("last_fetched", "")
+    last_fetched = parse_last_fetched_date(last_fetched_str, first_fetch_time)
+    last_fetched_str = last_fetched.strftime(DATE_FORMAT)
+
+    # Saved offset of last run
+    last_offset_str: str = last_run.get("last_offset", "0")
+    last_offset = int(last_offset_str)
+
+    # Date of first run
+    first_run_at_str = last_run.get("first_run_at", "")
+    first_run_at = parse_last_fetched_date(first_run_at_str, first_fetch_time)
+
+    # Last modified fetch date
+    last_modified_fetched_str = last_run.get("last_modified_fetched", "")
+    last_modified_fetched = parse_last_fetched_date(last_modified_fetched_str, first_fetch_time)
+    last_modified_fetched_str = last_modified_fetched.strftime(DATE_FORMAT)
+
+    # Saved modified alerts offset of last run
+    last_modified_offset_str: str = last_run.get("last_modified_offset", "0")
+    last_modified_offset = int(last_modified_offset_str)
+
+    # ZeroFox Alert IDs previously created
+    zf_ids: list[int] = last_run.get("zf-ids", [])
+
+    next_run: FetchIncidentsStorage = {
+        "last_fetched": last_fetched_str,
+        "last_offset": last_offset_str,
+        "first_run_at": first_run_at.strftime(DATE_FORMAT),
+        "last_modified_fetched": last_modified_fetched_str,
+        "last_modified_offset": last_modified_offset_str,
+        "zf-ids": zf_ids,
+    }
+
+    # Fetch new alerts
+    params = {
+        "min_timestamp": last_fetched.strftime(DATE_FORMAT),
+        "sort_direction": "asc",
+        "offset": last_offset,
+    }
+    incidents, next_offset, oldest_timestamp, alert_ids = get_incidents_data(
+        client=client,
+        params=params,
+    )
+    if len(incidents) > 0:
+        ingested_alert_ids = alert_ids + zf_ids
+        next_run["zf-ids"] = ingested_alert_ids[:MAX_ALERT_IDS_STORED]
+        next_run["last_offset"] = next_offset
+        if next_offset == "0" and oldest_timestamp:
+            next_run["last_fetched"] = oldest_timestamp
+        return next_run, incidents
+
+    # If no new alerts, fetch modified alerts
+    params = {
+        "last_modified_min_date": last_modified_fetched.strftime(DATE_FORMAT),
+        "sort_direction": "asc",
+        "offset": last_modified_offset,
+    }
+
+    def is_not_a_new_alert(alert):
+        return alert.get("id") not in zf_ids
+    incidents, next_offset, oldest_timestamp, alert_ids = get_incidents_data(
+        client=client,
+        params=params,
+        is_valid_alert=is_not_a_new_alert,
+        timestamp_field="last_modified",
+    )
+    if len(incidents) > 0:
+        ingested_alert_ids = alert_ids + zf_ids
+        next_run["zf-ids"] = ingested_alert_ids[:MAX_ALERT_IDS_STORED]
+        next_run["last_modified_offset"] = next_offset
+        if next_offset == "0" and oldest_timestamp:
+            next_run["last_modified_fetched"] = oldest_timestamp
+        return next_run, incidents
 
-    return next_run, incidents
+    return next_run, []
 
 
 def get_modified_remote_data_command(

Packs/ZeroFox/Integrations/ZeroFox/ZeroFox.yml

@@ -788,7 +788,7 @@ script:
     deprecated: false
     execution: false
   - name: zerofox-search-compromised-domain
-    description: Looks for a given domain in Zerofox's CTI feeds
+    description: Looks for a given domain in Zerofox's CTI feeds.
     arguments:
     - name: domain
       required: true
@@ -805,7 +805,7 @@ script:
       type: string
       description: Related domains to the threat separated by commas.
   - name: zerofox-search-compromised-email
-    description: Looks for a given email in ZeroFox's CTI feeds
+    description: Looks for a given email in ZeroFox's CTI feeds.
     arguments:
     - name: email
       required: true
@@ -823,7 +823,7 @@ script:
       type: string
       description: Date in which the email was found related to a threat.
   - name: zerofox-search-malicious-ip
-    description: Looks for malicious ips in ZeroFox's CTI feeds
+    description: Looks for malicious ips in ZeroFox's CTI feeds.
     arguments:
     - name: ip
       required: true
@@ -841,7 +841,7 @@ script:
       type: string
       description: Date in which the ip was found related to a threat.
   - name: zerofox-search-malicious-hash
-    description: Looks for registered hashes in ZeroFox's CTI feeds
+    description: Looks for registered hashes in ZeroFox's CTI feeds.
     arguments:
     - name: hash
       required: true