From 761eb3fbb8417f542ea0091d6a58e7d6278f9e21 Mon Sep 17 00:00:00 2001 From: yucohen Date: Sun, 1 Dec 2024 19:14:36 +0200 Subject: [PATCH 1/5] improve netskope --- .../NetskopeEventCollector.py | 50 +++++++++++-------- .../NetskopeEventCollector_test.py | 12 +++-- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py index e752ebc1156d..0595a855bacf 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py @@ -185,7 +185,7 @@ def setup_last_run(last_run_dict: dict, event_types_to_fetch: list[str]) -> dict def handle_data_export_single_event_type(client: Client, event_type: str, operation: str, limit: int, - execution_start_time: datetime) -> tuple[list, bool]: + execution_start_time: datetime, all_event_types: list) -> bool: """ Pulls events per each given event type. Each event type receives a dedicated index name that is constructed using the event type and the integration instance name. The function keeps pulling events as long as the limit was not exceeded. @@ -216,12 +216,12 @@ def handle_data_export_single_event_type(client: Client, event_type: str, operat # If the execution exceeded the timeout we will break if is_execution_time_exceeded(start_time=execution_start_time): - return events, True + return True # Wait time between queries if wait_time: demisto.debug(f'Going to sleep between queries, wait_time is {wait_time} seconds') - time.sleep(wait_time) # pylint: disable=E9003 + time.sleep(wait_time) # pylint: disable=E9003 else: demisto.debug('No wait time received, going to sleep for 1 second') time.sleep(1) @@ -239,16 +239,18 @@ def handle_data_export_single_event_type(client: Client, event_type: str, operat events.extend(results) + all_event_types.extend(prepare_events(results, event_type)) + if not results or len(results) < MAX_EVENTS_PAGE_SIZE: break print_event_statistics_logs(events=events, event_type=event_type) # We mark this event type as successfully fetched client.fetch_status[event_type] = True - return events, False + return False -def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_SIZE) -> tuple[list, dict]: +def get_all_events(client: Client, last_run: dict, all_event_types: list, limit: int = MAX_EVENTS_PAGE_SIZE) -> dict: """ Iterates over all supported event types and call the handle data export logic. Once each event type is done the operation for next run is set to 'next'. @@ -263,35 +265,35 @@ def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_ dict: The updated last_run object. """ - all_types_events_result = [] + # all_event_types = [] execution_start_time = datetime.utcnow() for event_type in client.event_types_to_fetch: event_type_operation = last_run.get(event_type, {}).get('operation') - events, time_out = handle_data_export_single_event_type(client=client, event_type=event_type, - operation=event_type_operation, limit=limit, - execution_start_time=execution_start_time) - all_types_events_result.extend(prepare_events(events, event_type)) + time_out = handle_data_export_single_event_type(client=client, event_type=event_type, + operation=event_type_operation, limit=limit, + execution_start_time=execution_start_time, + all_event_types=all_event_types) last_run[event_type] = {'operation': 'next'} if time_out: demisto.info('Timeout reached, stopped pulling events') break - return all_types_events_result, last_run + return last_run ''' COMMAND FUNCTIONS ''' def test_module(client: Client, last_run: dict, max_fetch: int) -> str: - get_all_events(client, last_run, limit=max_fetch, ) + get_all_events(client, last_run, limit=max_fetch, all_event_types=[]) return 'ok' -def get_events_command(client: Client, args: dict[str, Any], last_run: dict) -> tuple[CommandResults, list]: +def get_events_command(client: Client, args: dict[str, Any], last_run: dict, events: list) -> tuple[CommandResults, list]: limit = arg_to_number(args.get('limit')) or MAX_EVENTS_PAGE_SIZE - events, _ = get_all_events(client=client, last_run=last_run, limit=limit) + _ = get_all_events(client=client, last_run=last_run, limit=limit, all_event_types=events) for event in events: event['timestamp'] = timestamp_to_datestring(event['timestamp'] * 1000) @@ -357,7 +359,8 @@ def main() -> None: # pragma: no cover last_run = setup_last_run(demisto.getLastRun(), event_types_to_fetch) demisto.debug(f'Running with the following last_run - {last_run}') - events: list[dict] = [] + # events: list[dict] = [] + all_event_types: list[dict] = [] new_last_run: dict = {} if command_name == 'test-module': # This is the call made when pressing the integration Test button. @@ -365,9 +368,10 @@ def main() -> None: # pragma: no cover return_results(result) elif command_name == 'netskope-get-events': - results, events = get_events_command(client, demisto.args(), last_run) + results, events = get_events_command(client, demisto.args(), last_run, events=[]) if argToBoolean(demisto.args().get('should_push_events', 'true')): - send_events_to_xsiam(events=events, vendor=vendor, product=product) # type: ignore + send_events_to_xsiam(events=events, vendor=vendor, product=product, + chunk_size=XSIAM_EVENT_CHUNK_SIZE_LIMIT) # type: ignore return_results(results) elif command_name == 'fetch-events': @@ -375,10 +379,12 @@ def main() -> None: # pragma: no cover start = datetime.utcnow() try: demisto.debug(f'Sending request with last run {last_run}') - events, new_last_run = get_all_events(client, last_run, max_fetch) + new_last_run = get_all_events(client=client, last_run=last_run, limit=max_fetch, + all_event_types=all_event_types) finally: - demisto.debug(f'sending {len(events)} to xsiam') - send_events_to_xsiam(events=events, vendor=vendor, product=product) + demisto.debug(f'sending {len(all_event_types)} to xsiam') + send_events_to_xsiam(events=all_event_types, vendor=vendor, product=product, + chunk_size=XSIAM_EVENT_CHUNK_SIZE_LIMIT) for event_type, status, in client.fetch_status.items(): if not status: @@ -386,8 +392,8 @@ def main() -> None: # pragma: no cover end = datetime.utcnow() - demisto.debug(f'Handled {len(events)} total events in {(end - start).seconds} seconds') - next_trigger_time(len(events), max_fetch, new_last_run) + demisto.debug(f'Handled {len(all_event_types)} total events in {(end - start).seconds} seconds') + next_trigger_time(len(all_event_types), max_fetch, new_last_run) demisto.debug(f'Setting the last_run to: {new_last_run}') demisto.setLastRun(new_last_run) diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_test.py b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_test.py index db861d6c9ece..2124998855ff 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_test.py +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_test.py @@ -75,7 +75,8 @@ def json_callback(request, _): proxy=False, event_types_to_fetch=ALL_SUPPORTED_EVENT_TYPES) url_matcher = re.compile('https://netskope[.]example[.]com/events/dataexport/events') requests_mock.get(url_matcher, json=json_callback) - events, new_last_run = get_all_events(client, FIRST_LAST_RUN) + events = [] + new_last_run = get_all_events(client, FIRST_LAST_RUN, all_event_types=events) assert len(events) == 26 assert events[0].get('event_id') == '1' assert events[0].get('_time') == '2023-05-22T10:30:16.000Z' @@ -95,9 +96,9 @@ def test_get_events_command(mocker): """ from NetskopeEventCollector import get_events_command client = Client(BASE_URL, 'dummy_token', False, False, event_types_to_fetch=ALL_SUPPORTED_EVENT_TYPES) - mocker.patch('NetskopeEventCollector.get_all_events', return_value=[MOCK_ENTRY, {}]) + mocker.patch('NetskopeEventCollector.get_all_events', return_value={}) mocker.patch.object(time, "sleep") - results, events = get_events_command(client, args={}, last_run=FIRST_LAST_RUN) + results, events = get_events_command(client, args={}, last_run=FIRST_LAST_RUN, events=MOCK_ENTRY) assert 'Events List' in results.readable_output assert len(events) == 9 assert results.outputs_prefix == 'Netskope.Event' @@ -274,9 +275,10 @@ def test_incident_endpoint(mocker): mocker.patch('NetskopeEventCollector.print_event_statistics_logs') client = Client(BASE_URL, 'dummy_token', False, False, event_types_to_fetch=['incident']) mock_response = MagicMock() - mock_response.json.return_value = {'result': 'fake_result', 'wait_time': 0} + mock_response.json.return_value = {'result': EVENTS_RAW['result'], 'wait_time': 0} request_mock = mocker.patch.object(Client, '_http_request', return_value=mock_response) - handle_data_export_single_event_type(client, 'incident', 'next', limit=50, execution_start_time=datetime.now()) + handle_data_export_single_event_type(client, 'incident', 'next', limit=50, + execution_start_time=datetime.now(), all_event_types=[]) kwargs = request_mock.call_args.kwargs assert kwargs['url_suffix'] == 'events/dataexport/events/incident' assert kwargs['params'] == {'index': 'xsoar_collector_test_instance_incident', 'operation': 'next'} From d4aeb1b5d4dc4fdd22ad1b61461fd713a83d31ca Mon Sep 17 00:00:00 2001 From: yucohen Date: Sun, 1 Dec 2024 19:45:56 +0200 Subject: [PATCH 2/5] rn --- Packs/Netskope/ReleaseNotes/4_0_2.md | 6 ++++++ Packs/Netskope/pack_metadata.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Packs/Netskope/ReleaseNotes/4_0_2.md diff --git a/Packs/Netskope/ReleaseNotes/4_0_2.md b/Packs/Netskope/ReleaseNotes/4_0_2.md new file mode 100644 index 000000000000..c44f84c2778f --- /dev/null +++ b/Packs/Netskope/ReleaseNotes/4_0_2.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Netskope Event Collector + +- Fixed an issue where in rare cases not all events were created. diff --git a/Packs/Netskope/pack_metadata.json b/Packs/Netskope/pack_metadata.json index 7edd223200ee..844be8e42e7d 100644 --- a/Packs/Netskope/pack_metadata.json +++ b/Packs/Netskope/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Netskope", "description": "Cloud access security broker that enables to find, understand, and secure cloud apps.", "support": "xsoar", - "currentVersion": "4.0.1", + "currentVersion": "4.0.2", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 1ea891a4558adba311b43a4db36ec43cc9525637 Mon Sep 17 00:00:00 2001 From: yucohen Date: Mon, 23 Dec 2024 12:45:43 +0200 Subject: [PATCH 3/5] RN and docs --- .../NetskopeEventCollector_description.md | 2 +- .../Netskope/Integrations/NetskopeEventCollector/README.md | 2 +- Packs/Netskope/ReleaseNotes/4_0_3.md | 7 +++++++ Packs/Netskope/pack_metadata.json | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Packs/Netskope/ReleaseNotes/4_0_3.md diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md index 4db1f7a8dc50..6c69d4d7eada 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md @@ -8,7 +8,7 @@ - Network - Alert - Page -- Note: The collector can handle 10K events per minute on average per each event type. +- Note: The collector can handle 35K events per minute on average per each event type. ### API Key - To generate the API token, in your Netskope UI go to **Settings** > **Tools** > **Rest API v2** diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/README.md b/Packs/Netskope/Integrations/NetskopeEventCollector/README.md index 8ab9ac77e322..a4759f26075e 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/README.md +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/README.md @@ -14,7 +14,7 @@ This is the default integration for this content pack when configured by the Dat ## Fetch Events Limitation -The collector can handle 10K events per minute on average per each event type +The collector can handle 35K events per minute on average per each event type ## Commands diff --git a/Packs/Netskope/ReleaseNotes/4_0_3.md b/Packs/Netskope/ReleaseNotes/4_0_3.md new file mode 100644 index 000000000000..21d3012af9ac --- /dev/null +++ b/Packs/Netskope/ReleaseNotes/4_0_3.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### Netskope Event Collector + +- Fixed an issue where in rare cases not all events were created. + diff --git a/Packs/Netskope/pack_metadata.json b/Packs/Netskope/pack_metadata.json index 844be8e42e7d..182f12bf100f 100644 --- a/Packs/Netskope/pack_metadata.json +++ b/Packs/Netskope/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Netskope", "description": "Cloud access security broker that enables to find, understand, and secure cloud apps.", "support": "xsoar", - "currentVersion": "4.0.2", + "currentVersion": "4.0.3", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 1eed5864b6cbe6e52ceae77c45ba5ecb3da084ce Mon Sep 17 00:00:00 2001 From: yucohen Date: Mon, 23 Dec 2024 15:28:33 +0200 Subject: [PATCH 4/5] CR --- .../NetskopeEventCollector/NetskopeEventCollector.py | 1 - .../NetskopeEventCollector_description.md | 2 +- Packs/Netskope/Integrations/NetskopeEventCollector/README.md | 2 +- Packs/Netskope/ReleaseNotes/4_0_3.md | 3 +-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py index 0595a855bacf..ff0373ffcb8a 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py @@ -359,7 +359,6 @@ def main() -> None: # pragma: no cover last_run = setup_last_run(demisto.getLastRun(), event_types_to_fetch) demisto.debug(f'Running with the following last_run - {last_run}') - # events: list[dict] = [] all_event_types: list[dict] = [] new_last_run: dict = {} if command_name == 'test-module': diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md index 6c69d4d7eada..7f0633209b52 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_description.md @@ -8,7 +8,7 @@ - Network - Alert - Page -- Note: The collector can handle 35K events per minute on average per each event type. +- Note: The collector can handle up to 35K events per minute on average. ### API Key - To generate the API token, in your Netskope UI go to **Settings** > **Tools** > **Rest API v2** diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/README.md b/Packs/Netskope/Integrations/NetskopeEventCollector/README.md index a4759f26075e..de6058486d46 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/README.md +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/README.md @@ -14,7 +14,7 @@ This is the default integration for this content pack when configured by the Dat ## Fetch Events Limitation -The collector can handle 35K events per minute on average per each event type +The collector can handle up to 35K events per minute on average. ## Commands diff --git a/Packs/Netskope/ReleaseNotes/4_0_3.md b/Packs/Netskope/ReleaseNotes/4_0_3.md index 21d3012af9ac..85a3f9aa2e21 100644 --- a/Packs/Netskope/ReleaseNotes/4_0_3.md +++ b/Packs/Netskope/ReleaseNotes/4_0_3.md @@ -3,5 +3,4 @@ ##### Netskope Event Collector -- Fixed an issue where in rare cases not all events were created. - +- Fixed an issue where in rare cases not all events were pulled. From e140a8c0fe37326dadaf9e717f2b309973ddff49 Mon Sep 17 00:00:00 2001 From: yucohen Date: Mon, 23 Dec 2024 17:17:00 +0200 Subject: [PATCH 5/5] CR --- .../NetskopeEventCollector/NetskopeEventCollector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py index ff0373ffcb8a..522f1e35916a 100644 --- a/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py +++ b/Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py @@ -265,7 +265,6 @@ def get_all_events(client: Client, last_run: dict, all_event_types: list, limit: dict: The updated last_run object. """ - # all_event_types = [] execution_start_time = datetime.utcnow() for event_type in client.event_types_to_fetch: event_type_operation = last_run.get(event_type, {}).get('operation')