add changelog entry and first draft of implementation

CHANGELOG.md

@@ -4,6 +4,9 @@
 ### Breaking
 ### Features
 ### Improvenets
+
+* predetector now normalizes timestamps so they can be read correctly by opensearch
+
 ### Bugfix
 
 ## 13.0.1

logprep/processor/pre_detector/processor.py

@@ -30,6 +30,7 @@
 .. automodule:: logprep.processor.pre_detector.rule
 """
 
+from datetime import datetime
 from functools import cached_property
 from uuid import uuid4
 
@@ -92,6 +93,14 @@ class Config(Processor.Config):
     def _ip_alerter(self):
         return IPAlerter(self._config.alert_ip_list_path)
 
+    def is_normalized_timestamp(self, timestamp: str):
+        """this method checks if the timestamp has been normalized"""
+        try:
+            datetime.fromisoformat(timestamp)
+            return True
+        except ValueError:
+            return False
+
     def _apply_rules(self, event, rule):
         if not (
             self._ip_alerter.has_ip_fields(rule)
@@ -101,8 +110,15 @@ def _apply_rules(self, event, rule):
         for detection, _ in self.result.data:
             detection["creation_timestamp"] = TimeParser.now().isoformat()
             timestamp = get_dotted_field_value(event, "@timestamp")
+
             if timestamp is not None:
-                detection["@timestamp"] = timestamp
+                if self.is_normalized_timestamp(timestamp):
+                    detection["@timestamp"] = timestamp
+                else:
+                    # need to find out how to get every format not just unix..
+                    timestamp = TimeParser.parse_datetime(timestamp, "UNIX", "UTC")
+                    result = timestamp.isoformat()
+                    detection["@timestamp"] = result
 
     def _get_detection_result(self, event: dict, rule: PreDetectorRule):
         pre_detection_id = get_dotted_field_value(event, "pre_detection_id")

tests/unit/processor/pre_detector/test_pre_detector.py

@@ -329,8 +329,16 @@ def _assert_equality_of_results(
 
     def test_adds_timestamp_to_extra_data_if_provided_by_event(self):
         document = {
-            "@timestamp": "custom timestamp",
+            "@timestamp": "2024-08-12T12:13:04Z",
             "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}},
         }
         detection_results = self.object.process(document)
-        assert detection_results.data[0][0].get("@timestamp") == "custom timestamp"
+        assert detection_results.data[0][0].get("@timestamp") == "2024-08-12T12:13:04Z"
+
+    def test_timestamp_is_normalised(self):
+        document = {
+            "@timestamp": "1723464784",
+            "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}},
+        }
+        detection_results = self.object.process(document)
+        assert detection_results.data[0][0].get("@timestamp") == "2024-08-12T12:13:04+00:00"