From 0ee0d6b4300830d5d470055b599f12ac4ca63d2c Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 22 Aug 2023 12:51:18 +0200 Subject: [PATCH 1/5] Add creation timestamp to pre-detections --- logprep/processor/pre_detector/processor.py | 3 +++ tests/unit/processor/pre_detector/test_pre_detector.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/logprep/processor/pre_detector/processor.py b/logprep/processor/pre_detector/processor.py index f4156e167..3fce3c4ed 100644 --- a/logprep/processor/pre_detector/processor.py +++ b/logprep/processor/pre_detector/processor.py @@ -38,6 +38,7 @@ from logprep.abc.processor import Processor from logprep.processor.pre_detector.ip_alerter import IPAlerter from logprep.processor.pre_detector.rule import PreDetectorRule +from logprep.util.time import TimeParser class PreDetectorError(BaseException): @@ -128,6 +129,8 @@ def _apply_rules(self, event, rule): if "@timestamp" in event: for detection in self._extra_data: detection["@timestamp"] = event["@timestamp"] + for detection in self._extra_data: + detection["creation_timestamp"] = TimeParser.now().isoformat() def _get_detection_result(self, rule: PreDetectorRule, detection_results: list): if self._event.get("pre_detection_id") is None: diff --git a/tests/unit/processor/pre_detector/test_pre_detector.py b/tests/unit/processor/pre_detector/test_pre_detector.py index 73e68b2e4..e287fc7b0 100644 --- a/tests/unit/processor/pre_detector/test_pre_detector.py +++ b/tests/unit/processor/pre_detector/test_pre_detector.py @@ -287,6 +287,9 @@ def test_ignores_case_list(self): def _assert_equality_of_results( self, document, expected, detection_results, expected_detection_results ): + for detection_result in detection_results[0]: + assert detection_result.pop("creation_timestamp") + pre_detection_id = document.pop("pre_detection_id", None) assert pre_detection_id is not None From df2a3c75a29cdf0f2838e68693b66d40a6264379 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Fri, 25 Aug 2023 14:32:41 +0200 Subject: [PATCH 2/5] Fix pre_detection acceptance test --- tests/acceptance/test_pre_detection.py | 27 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/acceptance/test_pre_detection.py b/tests/acceptance/test_pre_detection.py index fdbd2657c..e20900bf6 100644 --- a/tests/acceptance/test_pre_detection.py +++ b/tests/acceptance/test_pre_detection.py @@ -1,5 +1,6 @@ # pylint: disable=missing-docstring # pylint: disable=line-too-long +# pylint: disable=too-many-locals import json import re from logging import basicConfig, DEBUG, getLogger @@ -39,28 +40,28 @@ ( {"@timestamp":"2019-07-30T14:38:16.352Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"event":{"code":7036,"created":"2019-08-02T09:55:11.996Z","kind":"event"},"agent":{"version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0"},"ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4"},"log":{"level":"information"},"message":"The Software Protection service entered the stopped state.","winlog":{"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_id":1234,"task":"","api":"wineventlog","event_data":{"Binary":"7300700070007300760063002F0031000000","param1":"Software Protection","param2":"stopped"},"keywords":["Classic"],"provider_name":"Service Control Manager","record_id":11580,"channel":"System","computer_name":"abcdefg1234","process":{"thread":{"id":2808},"pid":436}}}, {"@timestamp":"2019-07-30T14:38:16.352Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"event":{"code":7036,"created":"2019-08-02T09:55:11.996Z","kind":"event"},"agent":{"version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0"},"ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4"},"log":{"level":"information"},"message":"The Software Protection service entered the stopped state.","winlog":{"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_id":1234,"task":"","api":"wineventlog","event_data":{"Binary":"7300700070007300760063002F0031000000","param1":"Software Protection","param2":"stopped"},"keywords":["Classic"],"provider_name":"Service Control Manager","record_id":11580,"channel":"System","computer_name":"abcdefg1234","process":{"thread":{"id":2808},"pid":436}}}, - [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "1cf39644-a632-4c42-a7b4-2896c4efffb5", "host": {"name": "CLIENT1"}, "@timestamp": "2019-07-30T14:38:16.352Z"}}] + [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "1cf39644-a632-4c42-a7b4-2896c4efffb5", "host": {"name": "CLIENT1"}, "@timestamp": "2019-07-30T14:38:16.352Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}] ), ( {"@timestamp":"2019-08-02T09:46:41.906Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"host":{"name":"CLIENT1","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1","architecture":"x86"},"agent":{"hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9"},"ecs":{"version":"1.0.0"},"winlog":{"channel":"System","provider_name":"Service Control Manager","record_id":11627,"event_id":1234,"api":"wineventlog","keywords":["Classic"],"computer_name":"abcdefg1234","process":{"pid":440,"thread":{"id":524}},"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_data":{"param1":"Wazuh","param2":"running","Binary":"4F0073007300650063005300760063002F0034000000"},"task":""},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:11.998Z"},"log":{"level":"information"},"message":"The Wazuh service entered the running state."}, {"@timestamp":"2019-08-02T09:46:41.906Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"host":{"name":"CLIENT1","os":{"name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1","architecture":"x86"},"agent":{"hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9"},"ecs":{"version":"1.0.0"},"winlog":{"channel":"System","provider_name":"Service Control Manager","record_id":11627,"event_id":1234,"api":"wineventlog","keywords":["Classic"],"computer_name":"abcdefg1234","process":{"pid":440,"thread":{"id":524}},"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","event_data":{"param1":"Wazuh","param2":"running","Binary":"4F0073007300650063005300760063002F0034000000"},"task":""},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:11.998Z"},"log":{"level":"information"},"message":"The Wazuh service entered the running state."}, - [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "08d1aa6f-f508-464e-a13d-0b5da46b5bcc", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:41.906Z"}}] + [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "08d1aa6f-f508-464e-a13d-0b5da46b5bcc", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:41.906Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}] ), ( {"@timestamp":"2019-08-02T09:46:54.583Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"provider_name":"Service Control Manager","computer_name":"abcdefg1234","provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","process":{"pid":440,"thread":{"id":1792}},"event_data":{"param1":"Portable Device Enumerator Service","param2":"running","Binary":"57005000440042007500730045006E0075006D002F0034000000"},"channel":"System","record_id":11638,"task":"","api":"wineventlog","event_id":1234,"keywords":["Classic"]},"event":{"code":7036,"created":"2019-08-02T09:55:11.999Z","kind":"event"},"agent":{"ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat"},"ecs":{"version":"1.0.0"},"host":{"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0"}},"log":{"level":"information"},"message":"The Portable Device Enumerator Service service entered the running state."}, {"@timestamp":"2019-08-02T09:46:54.583Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"provider_name":"Service Control Manager","computer_name":"abcdefg1234","provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","process":{"pid":440,"thread":{"id":1792}},"event_data":{"param1":"Portable Device Enumerator Service","param2":"running","Binary":"57005000440042007500730045006E0075006D002F0034000000"},"channel":"System","record_id":11638,"task":"","api":"wineventlog","event_id":1234,"keywords":["Classic"]},"event":{"code":7036,"created":"2019-08-02T09:55:11.999Z","kind":"event"},"agent":{"ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1","id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat"},"ecs":{"version":"1.0.0"},"host":{"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","name":"CLIENT1","hostname":"CLIENT1","architecture":"x86","os":{"platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional","kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0"}},"log":{"level":"information"},"message":"The Portable Device Enumerator Service service entered the running state."}, - [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "06d12743-01f0-4793-8a31-3815cfa31fc3", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:54.583Z"}}] + [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "06d12743-01f0-4793-8a31-3815cfa31fc3", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:46:54.583Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}] ), ( {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager 2","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}}, {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager 2","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}}, - [{"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}}] + [{"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}] ), ( {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}}, {"@timestamp":"2019-08-02T09:54:57.125Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.2.0","topic":"wineventlog_raw"},"winlog":{"computer_name":"abcdefg1234","event_id":123,"record_id":11714,"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","task":"","api":"wineventlog","event_data":{"param2":"running","Binary":"41007500640069006F007300720076002F0034000000","param1":"Windows Audio"},"provider_name":"Service Control Manager","keywords":["Classic"],"process":{"pid":440,"thread":{"id":528}}},"event":{"kind":"event","code":7036,"created":"2019-08-02T09:55:12.091Z"},"log":{"level":"information"},"message":"The Windows Audio service entered the running state.","ecs":{"version":"1.0.0"},"host":{"name":"CLIENT1","architecture":"x86","os":{"kernel":"6.1.7601.18741 (win7sp1_gdr.150202-1526)","build":"7601.0","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional"},"id":"19fc45ac-5890-4f96-81b1-50ed111c0ce4","hostname":"CLIENT1"},"agent":{"id":"0b755aca-0a9a-454a-9800-1979901962a0","version":"7.2.0","type":"winlogbeat","ephemeral_id":"de845cd9-5141-4c92-ad32-27a4518307e9","hostname":"CLIENT1"}}, [{"pre_detector_topic": {"description": "", "id": "RULE_ONE_ID", "title": "RULE_ONE", "severity": "critical", "mitre": ["mitre1", "mitre2"], "case_condition": "directly", "rule_filter": 'winlog.provider_name:"Service Control Manager"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}}, - {"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z"}}] + {"pre_detector_topic": {"description": "", "id": "RULE_TWO_ID", "title": "RULE_TWO", "severity": "critical", "mitre": ["mitre2", "mitre3"], "case_condition": "directly", "rule_filter": 'winlog.event_id:"123"', "pre_detection_id": "638cc0b3-b912-4220-8551-defea8ea139d", "host": {"name": "CLIENT1"}, "@timestamp": "2019-08-02T09:54:57.125Z", "creation_timestamp": "2019-07-30T14:58:16.352Z"}}] ), ], ) @@ -77,19 +78,27 @@ def test_events_pre_detected_correctly( logprep_output, logprep_extra_output, logprep_error_output = get_test_output(config_path) assert not logprep_error_output diff = DeepDiff( - expected_output_event, logprep_output[0], exclude_paths="root['pre_detection_id']" + expected_output_event, logprep_output[0], exclude_paths="root['pre_detection_id']" # pylint: disable=unsubscriptable-object ) assert not diff, f"The expected output event and the logprep output differ: {diff}" if expected_extra_output is not None: # compare every expected extra output with every logprep extra output and search for match for expected_extra_out in expected_extra_output: has_matching_output = False - for logprep_extra_out in logprep_extra_output: - exclude_regex_path = re.compile( + for logprep_extra_out in logprep_extra_output: # pylint: disable=not-an-iterable + exclude_pre_detection_id_regex_path = re.compile( r"root\['pre_detector_topic'\]\['pre_detection_id'\]" ) + exclude_creation_timestamp_regex_path = re.compile( + r"root\['pre_detector_topic'\]\['creation_timestamp'\]" + ) diff = DeepDiff( - expected_extra_out, logprep_extra_out, exclude_regex_paths=exclude_regex_path + expected_extra_out, + logprep_extra_out, + exclude_regex_paths=[ + exclude_pre_detection_id_regex_path, + exclude_creation_timestamp_regex_path, + ], ) if not diff: has_matching_output = True From f6db552bc07385fc541031cef69c649d3d9505d4 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Fri, 25 Aug 2023 14:37:28 +0200 Subject: [PATCH 3/5] Fix black formatting --- tests/acceptance/test_pre_detection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/test_pre_detection.py b/tests/acceptance/test_pre_detection.py index e20900bf6..abc3915c8 100644 --- a/tests/acceptance/test_pre_detection.py +++ b/tests/acceptance/test_pre_detection.py @@ -78,7 +78,9 @@ def test_events_pre_detected_correctly( logprep_output, logprep_extra_output, logprep_error_output = get_test_output(config_path) assert not logprep_error_output diff = DeepDiff( - expected_output_event, logprep_output[0], exclude_paths="root['pre_detection_id']" # pylint: disable=unsubscriptable-object + expected_output_event, + logprep_output[0], # pylint: disable=unsubscriptable-object + exclude_paths="root['pre_detection_id']", ) assert not diff, f"The expected output event and the logprep output differ: {diff}" if expected_extra_output is not None: From 56ce3f9956ad4f2a4685be690554cbfdb2dcf947 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Fri, 22 Sep 2023 07:33:36 +0200 Subject: [PATCH 4/5] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea4319b25..5c4cb0863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## next release ### Features ### Improvements + +* `pre_detector` processor now adds the field `creation_timestamp` to pre-detections. +It contains the time at which a pre-detection was created by the processor. + ### Bugfix ## v6.8.0 From f7a7497d0f7b7f6b9f322d12647b9e15f5f590ec Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Fri, 22 Sep 2023 07:34:47 +0200 Subject: [PATCH 5/5] Refactor pre_detector --- logprep/processor/pre_detector/processor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/logprep/processor/pre_detector/processor.py b/logprep/processor/pre_detector/processor.py index 3fce3c4ed..9364f58dd 100644 --- a/logprep/processor/pre_detector/processor.py +++ b/logprep/processor/pre_detector/processor.py @@ -126,11 +126,10 @@ def _apply_rules(self, event, rule): if self._logger.isEnabledFor(DEBUG): # pragma: no cover self._logger.debug(f"{self.describe()} processing matching event") self._get_detection_result(rule, self._extra_data) - if "@timestamp" in event: - for detection in self._extra_data: - detection["@timestamp"] = event["@timestamp"] for detection in self._extra_data: detection["creation_timestamp"] = TimeParser.now().isoformat() + if "@timestamp" in event: + detection["@timestamp"] = event["@timestamp"] def _get_detection_result(self, rule: PreDetectorRule, detection_results: list): if self._event.get("pre_detection_id") is None: