diff --git a/CHANGELOG.md b/CHANGELOG.md index 547431bbb..15eeb1b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Breaking ### Features ### Improvenets + +* predetector now normalizes timestamps so they can be read correctly by opensearch + ### Bugfix ## 13.0.1 diff --git a/logprep/processor/pre_detector/processor.py b/logprep/processor/pre_detector/processor.py index c44b32322..fbbd9bd37 100644 --- a/logprep/processor/pre_detector/processor.py +++ b/logprep/processor/pre_detector/processor.py @@ -30,6 +30,7 @@ .. automodule:: logprep.processor.pre_detector.rule """ +from datetime import datetime from functools import cached_property from uuid import uuid4 @@ -92,6 +93,14 @@ class Config(Processor.Config): def _ip_alerter(self): return IPAlerter(self._config.alert_ip_list_path) + def is_normalized_timestamp(self, timestamp: str): + """this method checks if the timestamp has been normalized""" + try: + datetime.fromisoformat(timestamp) + return True + except ValueError: + return False + def _apply_rules(self, event, rule): if not ( self._ip_alerter.has_ip_fields(rule) @@ -101,8 +110,15 @@ def _apply_rules(self, event, rule): for detection, _ in self.result.data: detection["creation_timestamp"] = TimeParser.now().isoformat() timestamp = get_dotted_field_value(event, "@timestamp") + if timestamp is not None: - detection["@timestamp"] = timestamp + if self.is_normalized_timestamp(timestamp): + detection["@timestamp"] = timestamp + else: + # need to find out how to get every format not just unix.. + timestamp = TimeParser.parse_datetime(timestamp, "UNIX", "UTC") + result = timestamp.isoformat() + detection["@timestamp"] = result def _get_detection_result(self, event: dict, rule: PreDetectorRule): pre_detection_id = get_dotted_field_value(event, "pre_detection_id") diff --git a/tests/unit/processor/pre_detector/test_pre_detector.py b/tests/unit/processor/pre_detector/test_pre_detector.py index 53da723cd..8305832c8 100644 --- a/tests/unit/processor/pre_detector/test_pre_detector.py +++ b/tests/unit/processor/pre_detector/test_pre_detector.py @@ -329,8 +329,16 @@ def _assert_equality_of_results( def test_adds_timestamp_to_extra_data_if_provided_by_event(self): document = { - "@timestamp": "custom timestamp", + "@timestamp": "2024-08-12T12:13:04Z", "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}}, } detection_results = self.object.process(document) - assert detection_results.data[0][0].get("@timestamp") == "custom timestamp" + assert detection_results.data[0][0].get("@timestamp") == "2024-08-12T12:13:04Z" + + def test_timestamp_is_normalised(self): + document = { + "@timestamp": "1723464784", + "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}}, + } + detection_results = self.object.process(document) + assert detection_results.data[0][0].get("@timestamp") == "2024-08-12T12:13:04+00:00"