Parse directly

Signed-off-by: Chase Engelbrecht <[email protected]>
engechas · Mar 18, 2024 · 410d17c · 410d17c
1 parent 20a2d01
commit 410d17c
Show file tree

Hide file tree

Showing 26 changed files with 430 additions and 244 deletions.
diff --git a/...ngine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java b/...ngine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java
@@ -17,7 +17,6 @@
 import org.opensearch.dataprepper.model.processor.Processor;
 import org.opensearch.dataprepper.model.record.Record;
 import org.opensearch.dataprepper.plugins.processor.converters.FindingConverter;
-import org.opensearch.dataprepper.plugins.processor.converters.OCSFConverter;
 import org.opensearch.dataprepper.plugins.processor.evaluator.RuleEvaluator;
 import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType;
 import org.opensearch.dataprepper.plugins.processor.model.matches.Match;
@@ -46,7 +45,6 @@ public class RuleEngineProcessor extends AbstractProcessor<Record<Event>, Record
 
     private final RuleEvaluator ruleEvaluator;
     private final RuleEngineProcessorConfig config;
-    private final OCSFConverter ocsfConverter;
     private final FindingConverter findingConverter;
     private final IndexManager indexManager;
     private final ExpressionEvaluator expressionEvaluator;
@@ -84,7 +82,6 @@ public RuleEngineProcessor(final PluginMetrics pluginMetrics,
         final RuleEngineConfig ruleEngineConfig = new RuleEngineConfig(config.getRuleRefreshInterval(), config.getLogFormat(), config.getLogType(),
                 config.getRuleSchema(), config.getRuleLocation());
         ruleEvaluator = ruleEngine.start(ruleEngineConfig);
-        ocsfConverter = new OCSFConverter();
         findingConverter = new FindingConverter();
         acknowledgementSet = null;
     }
@@ -99,7 +96,7 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
             acknowledgementSet = ((DefaultEventHandle) (records.iterator().next().getData().getEventHandle())).getAcknowledgementSet();
         }
 
-        final Map<String, DataType> idToData = convertToOCSF(records);
+        final Map<String, DataType> idToData = addTrackingData(records);
         final Collection<Match> dataWithMatches = ruleEvaluator.evaluate(idToData.values());
         final Collection<Record<Event>> matches = convertMatchesToEvents(dataWithMatches);
 
@@ -111,14 +108,14 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
         return records;
     }
 
-    private Map<String, DataType> convertToOCSF(final Collection<Record<Event>> records) {
+    private Map<String, DataType> addTrackingData(final Collection<Record<Event>> records) {
         return records.stream()
                 .map(record -> {
                     final String id = UUID.randomUUID().toString();
-                    final DataType ocsf = ocsfConverter.convert(id, record);
-                    ocsf.putMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record));
+                    final DataType dataType = (DataType) record.getData();
+                    dataType.putMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record));
 
-                    final Map.Entry<String, DataType> mapEntry = Map.entry(id, ocsf);
+                    final Map.Entry<String, DataType> mapEntry = Map.entry(id, dataType);
                     record.getData().put(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName(), id);
 
                     return mapEntry;

diff --git a/...c/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java b/...c/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java
@@ -43,11 +43,11 @@ private Map<String, Object> generateEventForMonitor(final Match match, final Lis
         eventMap.put("id", UUID.randomUUID().toString());
         eventMap.put("monitor_id", openSearchSigmaV1Rule.getMonitorId());
         eventMap.put("monitor_name", openSearchSigmaV1Rule.getDetectorName());
-        eventMap.put("index", match.getDataType().getMetadata().get(OpenSearchDocMetadata.INDEX.getFieldName()));
+        eventMap.put("index", match.getDataType().getMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName()));
         eventMap.put("queries", rules.stream().map(this::getQuery).collect(Collectors.toList()));
         eventMap.put("timestamp", Instant.now().toEpochMilli());
         eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_ID_REPLACEMENT_FIELDS.getFieldName(), List.of("related_doc_ids", "correlated_doc_ids"));
-        eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getMetadata().get(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName()));
+        eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getMetadataValue(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName()));
         eventMap.put(OpenSearchDocMetadata.FINDINGS_INDEX_NAME.getFieldName(), openSearchSigmaV1Rule.getFindingsIndex());
 
         return eventMap;

diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java b/.../src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java
diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java b/.../src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java
@@ -1,17 +1,31 @@
 package org.opensearch.dataprepper.plugins.processor.model.datatypes;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.opensearch.dataprepper.expression.ExpressionEvaluator;
+import org.opensearch.dataprepper.model.event.DefaultEventHandle;
+import org.opensearch.dataprepper.model.event.DefaultEventMetadata;
+import org.opensearch.dataprepper.model.event.Event;
+import org.opensearch.dataprepper.model.event.EventHandle;
+import org.opensearch.dataprepper.model.event.EventMetadata;
+import org.opensearch.dataprepper.model.event.EventType;
+
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-public abstract class DataType {
+public abstract class DataType implements Event {
+    @JsonIgnore
+    private final EventMetadata eventMetadata;
+    @JsonIgnore
     private final HashMap<String, String> metadata;
+    @JsonIgnore
+    private final transient EventHandle eventHandle;
 
     public DataType() {
+        eventMetadata = DefaultEventMetadata.builder().withEventType(EventType.LOG.toString()).build();
         metadata = new HashMap<>();
-    }
-
-    public HashMap<String, String> getMetadata() {
-        return metadata;
+        eventHandle = new DefaultEventHandle(eventMetadata.getTimeReceived());
     }
 
     public abstract Object getValue(final String fieldName);
@@ -27,4 +41,80 @@ public void putAllMetadata(final Map<String, String> metadataEntries) {
     public String getMetadataValue(final String metadataFieldName) {
         return metadata.get(metadataFieldName);
     }
+
+
+    @Override
+    public void put(String key, Object value) {
+
+    }
+
+    @Override
+    public <T> T get(String key, Class<T> clazz) {
+        return null;
+    }
+
+    @Override
+    public <T> List<T> getList(String key, Class<T> clazz) {
+        return null;
+    }
+
+    @Override
+    public void delete(String key) {
+
+    }
+
+    @Override
+    public String toJsonString() {
+        return null;
+    }
+
+    @Override
+    public JsonNode getJsonNode() {
+        return null;
+    }
+
+    @Override
+    public String getAsJsonString(String key) {
+        return null;
+    }
+
+    @Override
+    public EventMetadata getMetadata() {
+        return eventMetadata;
+    }
+
+    @Override
+    public boolean containsKey(String key) {
+        return false;
+    }
+
+    @Override
+    public boolean isValueAList(String key) {
+        return false;
+    }
+
+    @Override
+    public Map<String, Object> toMap() {
+        return null;
+    }
+
+    @Override
+    public String formatString(String format) {
+        return null;
+    }
+
+    @Override
+    public String formatString(String format, ExpressionEvaluator expressionEvaluator) {
+        return format;
+    }
+
+    @Override
+    public EventHandle getEventHandle() {
+        return eventHandle;
+    }
+
+    @Override
+    public JsonStringBuilder jsonBuilder() {
+        return null;
+    }
 }