From 410d17c592a045c5e9fa3ffec908b1243611f781 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Sun, 17 Mar 2024 19:13:42 -0700 Subject: [PATCH] Parse directly Signed-off-by: Chase Engelbrecht --- .../processor/RuleEngineProcessor.java | 13 +- .../converters/FindingConverter.java | 4 +- .../processor/converters/OCSFConverter.java | 67 -------- .../processor/model/datatypes/DataType.java | 100 ++++++++++- .../processor/model/datatypes/OCSF.java | 161 ------------------ .../processor/model/datatypes/ocsf/Actor.java | 13 ++ .../processor/model/datatypes/ocsf/Api.java | 12 ++ .../processor/model/datatypes/ocsf/Cloud.java | 9 + .../model/datatypes/ocsf/Endpoint.java | 10 ++ .../model/datatypes/ocsf/Feature.java | 8 + .../model/datatypes/ocsf/HttpRequest.java | 10 ++ .../processor/model/datatypes/ocsf/Idp.java | 8 + .../model/datatypes/ocsf/Metadata.java | 13 ++ .../processor/model/datatypes/ocsf/OCSF.java | 121 +++++++++++++ .../model/datatypes/ocsf/Product.java | 13 ++ .../model/datatypes/ocsf/Request.java | 8 + .../model/datatypes/ocsf/Resource.java | 12 ++ .../model/datatypes/ocsf/Response.java | 9 + .../model/datatypes/ocsf/Service.java | 8 + .../model/datatypes/ocsf/Session.java | 12 ++ .../processor/model/datatypes/ocsf/User.java | 16 ++ .../model/datatypes/{ => ocsf}/ocsf.json | 0 ...SigmaV1SigmaV1RuleConditionParserTest.java | 2 +- data-prepper-plugins/s3-source/build.gradle | 1 + .../plugins/source/s3/codec/OCSFCodec.java | 40 +++++ .../source/s3/codec/OCSFCodecConfig.java | 4 + 26 files changed, 430 insertions(+), 244 deletions(-) delete mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java delete mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/OCSF.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Actor.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Api.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Cloud.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Endpoint.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Feature.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/HttpRequest.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Idp.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Metadata.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Product.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Request.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Resource.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Response.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Service.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Session.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/User.java rename data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/{ => ocsf}/ocsf.json (100%) create mode 100644 data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodec.java create mode 100644 data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodecConfig.java diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java index daae9ef891..d8f7a1fa71 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java @@ -17,7 +17,6 @@ import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.processor.converters.FindingConverter; -import org.opensearch.dataprepper.plugins.processor.converters.OCSFConverter; import org.opensearch.dataprepper.plugins.processor.evaluator.RuleEvaluator; import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; import org.opensearch.dataprepper.plugins.processor.model.matches.Match; @@ -46,7 +45,6 @@ public class RuleEngineProcessor extends AbstractProcessor, Record private final RuleEvaluator ruleEvaluator; private final RuleEngineProcessorConfig config; - private final OCSFConverter ocsfConverter; private final FindingConverter findingConverter; private final IndexManager indexManager; private final ExpressionEvaluator expressionEvaluator; @@ -84,7 +82,6 @@ public RuleEngineProcessor(final PluginMetrics pluginMetrics, final RuleEngineConfig ruleEngineConfig = new RuleEngineConfig(config.getRuleRefreshInterval(), config.getLogFormat(), config.getLogType(), config.getRuleSchema(), config.getRuleLocation()); ruleEvaluator = ruleEngine.start(ruleEngineConfig); - ocsfConverter = new OCSFConverter(); findingConverter = new FindingConverter(); acknowledgementSet = null; } @@ -99,7 +96,7 @@ public Collection> doExecute(final Collection> recor acknowledgementSet = ((DefaultEventHandle) (records.iterator().next().getData().getEventHandle())).getAcknowledgementSet(); } - final Map idToData = convertToOCSF(records); + final Map idToData = addTrackingData(records); final Collection dataWithMatches = ruleEvaluator.evaluate(idToData.values()); final Collection> matches = convertMatchesToEvents(dataWithMatches); @@ -111,14 +108,14 @@ public Collection> doExecute(final Collection> recor return records; } - private Map convertToOCSF(final Collection> records) { + private Map addTrackingData(final Collection> records) { return records.stream() .map(record -> { final String id = UUID.randomUUID().toString(); - final DataType ocsf = ocsfConverter.convert(id, record); - ocsf.putMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record)); + final DataType dataType = (DataType) record.getData(); + dataType.putMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record)); - final Map.Entry mapEntry = Map.entry(id, ocsf); + final Map.Entry mapEntry = Map.entry(id, dataType); record.getData().put(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName(), id); return mapEntry; diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java index bd55f05864..570e85f4bb 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java @@ -43,11 +43,11 @@ private Map generateEventForMonitor(final Match match, final Lis eventMap.put("id", UUID.randomUUID().toString()); eventMap.put("monitor_id", openSearchSigmaV1Rule.getMonitorId()); eventMap.put("monitor_name", openSearchSigmaV1Rule.getDetectorName()); - eventMap.put("index", match.getDataType().getMetadata().get(OpenSearchDocMetadata.INDEX.getFieldName())); + eventMap.put("index", match.getDataType().getMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName())); eventMap.put("queries", rules.stream().map(this::getQuery).collect(Collectors.toList())); eventMap.put("timestamp", Instant.now().toEpochMilli()); eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_ID_REPLACEMENT_FIELDS.getFieldName(), List.of("related_doc_ids", "correlated_doc_ids")); - eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getMetadata().get(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName())); + eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getMetadataValue(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName())); eventMap.put(OpenSearchDocMetadata.FINDINGS_INDEX_NAME.getFieldName(), openSearchSigmaV1Rule.getFindingsIndex()); return eventMap; diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java deleted file mode 100644 index 9e84a75665..0000000000 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/OCSFConverter.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.opensearch.dataprepper.plugins.processor.converters; - -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.processor.model.datatypes.OCSF; -import org.opensearch.dataprepper.plugins.processor.util.OpenSearchDocMetadata; - -import java.util.Map; - -public class OCSFConverter { - public OCSF convert(final String id, final Record record) { - final Event event = record.getData(); - final OCSF ocsf = OCSF.builder() - .metadataProductVersion(event.get("/metadata/product/version", String.class)) - .metadataProductName(event.get("/metadata/product/name", String.class)) - .metadataProductVendorName(event.get("/metadata/product/vendor_name", String.class)) - .metadataProductFeatureName(event.get("/metadata/product/feature/name", String.class)) - .metadataUid(event.get("/metadata/uid", String.class)) - .metadataProfiles(event.getList("/metadata/profiles", String.class)) - .metadataVersion(event.get("/metadata/version", String.class)) - .time(event.get("/time", Long.class)) - .cloudRegion(event.get("/cloud/region", String.class)) - .cloudProvider(event.get("/cloud/provider", String.class)) - .dstEndpoint(event.get("/dst_endpoint", String.class)) - .httpRequestUserAgent(event.get("/http_request/user_agent", String.class)) - .srcEndpointUid(event.get("/src_endpoint/uid", String.class)) - .srcEndpointIp(event.get("/src_endpoint/ip", String.class)) - .srcEndpointDomain(event.get("/src_endpoint/domain", String.class)) - .className(event.get("/class_name", String.class)) - .classUid(event.get("/class_uid", Integer.class)) - .categoryName(event.get("/category_name", String.class)) - .categoryUid(event.get("/category_uid", Integer.class)) - .severityId(event.get("/severity_id", Integer.class)) - .severity(event.get("/severity", String.class)) - .user(event.get("/user", String.class)) - .activityName(event.get("/activity_name", String.class)) - .activityId(event.get("/activity_id", Integer.class)) - .typeUid(event.get("/type_uid", Integer.class)) - .typeName(event.get("/type_name", String.class)) - .status(event.get("/status", String.class)) - .statusId(event.get("/status_id", Integer.class)) - .mfa(event.get("/mfa", Boolean.class)) - .apiResponseError(event.get("/api/response/error", String.class)) - .apiResponseMessage(event.get("/api/response/message", String.class)) - .apiOperation(event.get("/api/operation", String.class)) - .apiVersion(event.get("/api/version", String.class)) - .apiServiceName(event.get("/api/service/name", String.class)) - .apiRequestUid(event.get("/api/request/uid", String.class)) - .resources(event.getList("/resources", OCSF.Resource.class)) - .actorUserType(event.get("/actor/user/type", String.class)) - .actorUserName(event.get("/actor/user/name", String.class)) - .actorUserUid(event.get("/actor/user/uid", String.class)) - .actorUserUuid(event.get("/actor/user/uuid", String.class)) - .actorUserAccountUid(event.get("/actor/user/account_uid", String.class)) - .actorUserCredentialUid(event.get("/actor/user/credential_uid", String.class)) - .actorSessionCreatedTime(event.get("/actor/session/created_time", Long.class)) - .actorSessionMfa(event.get("/actor/session/mfa", Boolean.class)) - .actorSessionIssuer(event.get("/actor/session/issuer", String.class)) - .actorInvokedBy(event.get("/actor/invoked_by", String.class)) - .actorIdpName(event.get("/actor/idp/name", String.class)) - .unmapped((Map) event.get("/unmapped", Map.class)) - .build(); - - ocsf.putMetadataValue(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName(), id); - return ocsf; - } -} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java index 415e485cc0..879e1949fb 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java @@ -1,17 +1,31 @@ package org.opensearch.dataprepper.plugins.processor.model.datatypes; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.model.event.DefaultEventHandle; +import org.opensearch.dataprepper.model.event.DefaultEventMetadata; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventHandle; +import org.opensearch.dataprepper.model.event.EventMetadata; +import org.opensearch.dataprepper.model.event.EventType; + import java.util.HashMap; +import java.util.List; import java.util.Map; -public abstract class DataType { +public abstract class DataType implements Event { + @JsonIgnore + private final EventMetadata eventMetadata; + @JsonIgnore private final HashMap metadata; + @JsonIgnore + private final transient EventHandle eventHandle; public DataType() { + eventMetadata = DefaultEventMetadata.builder().withEventType(EventType.LOG.toString()).build(); metadata = new HashMap<>(); - } - - public HashMap getMetadata() { - return metadata; + eventHandle = new DefaultEventHandle(eventMetadata.getTimeReceived()); } public abstract Object getValue(final String fieldName); @@ -27,4 +41,80 @@ public void putAllMetadata(final Map metadataEntries) { public String getMetadataValue(final String metadataFieldName) { return metadata.get(metadataFieldName); } + + + @Override + public void put(String key, Object value) { + + } + + @Override + public T get(String key, Class clazz) { + return null; + } + + @Override + public List getList(String key, Class clazz) { + return null; + } + + @Override + public void delete(String key) { + + } + + @Override + public String toJsonString() { + return null; + } + + @Override + public JsonNode getJsonNode() { + return null; + } + + @Override + public String getAsJsonString(String key) { + return null; + } + + @Override + public EventMetadata getMetadata() { + return eventMetadata; + } + + @Override + public boolean containsKey(String key) { + return false; + } + + @Override + public boolean isValueAList(String key) { + return false; + } + + @Override + public Map toMap() { + return null; + } + + @Override + public String formatString(String format) { + return null; + } + + @Override + public String formatString(String format, ExpressionEvaluator expressionEvaluator) { + return format; + } + + @Override + public EventHandle getEventHandle() { + return eventHandle; + } + + @Override + public JsonStringBuilder jsonBuilder() { + return null; + } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/OCSF.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/OCSF.java deleted file mode 100644 index b796eb4d51..0000000000 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/OCSF.java +++ /dev/null @@ -1,161 +0,0 @@ -package org.opensearch.dataprepper.plugins.processor.model.datatypes; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Builder; - -import java.util.List; -import java.util.Map; - -@Builder -public class OCSF extends DataType { - private final String metadataProductVersion; - private final String metadataProductName; - private final String metadataProductVendorName; - private final String metadataProductFeatureName; - private final String metadataUid; - private final List metadataProfiles; - private final String metadataVersion; - private final Long time; - private final String cloudRegion; - private final String cloudProvider; - private final String dstEndpoint; - private final String httpRequestUserAgent; - private final String srcEndpointUid; - private final String srcEndpointIp; - private final String srcEndpointDomain; - private final String className; - private final Integer classUid; - private final String categoryName; - private final Integer categoryUid; - private final Integer severityId; - private final String severity; - private final String user; - private final String activityName; - private final Integer activityId; - private final Integer typeUid; - private final String typeName; - private final String status; - private final Integer statusId; - private final Boolean mfa; - private final String apiResponseError; - private final String apiResponseMessage; - private final String apiOperation; - private final String apiVersion; - private final String apiServiceName; - private final String apiRequestUid; - private final List resources; - private final String actorUserType; - private final String actorUserName; - private final String actorUserUid; - private final String actorUserUuid; - private final String actorUserAccountUid; - private final String actorUserCredentialUid; - private final Long actorSessionCreatedTime; - private final Boolean actorSessionMfa; - private final String actorSessionIssuer; - private final String actorInvokedBy; - private final String actorIdpName; - private final Map unmapped; - - @Override - public Object getValue(final String fieldName) { - switch (fieldName) { - case "metadata.product.version": return metadataProductVersion; - case "metadata.product.name": return metadataProductName; - case "metadata.product.vendor_name": return metadataProductVendorName; - case "metadata.product.feature.name": return metadataProductFeatureName; - case "metadata.uid": return metadataUid; - case "metadata.profiles": return metadataProfiles; - case "metadata.version": return metadataVersion; - case "time": return time; - case "cloud.region": return cloudRegion; - case "cloud.provider": return cloudProvider; - case "dst_endpoint": return dstEndpoint; - case "http_request.user_agent": return httpRequestUserAgent; - case "src_endpoint.uid": return srcEndpointUid; - case "src_endpoint.ip": return srcEndpointIp; - case "src_endpoint.domain": return srcEndpointDomain; - case "class_name": return className; - case "class_uid": return classUid; - case "category_name": return categoryName; - case "category_uid": return categoryUid; - case "severity_id": return severityId; - case "severity": return severity; - case "user": return user; - case "activity_name": return activityName; - case "activity_id": return activityId; - case "type_uid": return typeUid; - case "type_name": return typeName; - case "status": return status; - case "status_id": return statusId; - case "mfa": return mfa; - case "api.response.error": return apiResponseError; - case "api.response.message": return apiResponseMessage; - case "api.operation": return apiOperation; - case "api.version": return apiVersion; - case "api.service.name": return apiServiceName; - case "api.request.uid": return apiRequestUid; - case "resources": return resources; - case "actor.user.type": return actorUserType; - case "actor.user.name": return actorUserName; - case "actor.user.uid": return actorUserUid; - case "actor.user.uuid": return actorUserUuid; - case "actor.user.account_uid": return actorUserAccountUid; - case "actor.user.credential_uid": return actorUserCredentialUid; - case "actor.session.created_time": return actorSessionCreatedTime; - case "actor.session.mfa": return actorSessionMfa; - case "actor.session.issuer": return actorSessionIssuer; - case "actor.invoked_by": return actorInvokedBy; - case "actor.idp.name": return actorIdpName; - case "unmapped": return unmapped; - default: return handleOtherFields(fieldName); - } - } - - private Object handleOtherFields(final String fieldName) { - final String[] parts = fieldName.split("\\."); - if (parts.length == 0 || !"unmapped".equals(parts[0])) { - throw new IllegalArgumentException("Field " + fieldName + " does not exist in class " + getClass().getName()); - } - - if (parts.length == 1) { - return unmapped; - } - - return unmapped.get(parts[1]); - } - - public static class Resource { - private String uid; - @JsonProperty("account_uid") - private String accountUid; - private String type; - - public Resource() { - } - - public String getUid() { - return uid; - } - - public void setUid(String uid) { - this.uid = uid; - } - - public String getAccountUid() { - return accountUid; - } - - public void setAccountUid(String accountUid) { - this.accountUid = accountUid; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - } -} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Actor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Actor.java new file mode 100644 index 0000000000..cd0f2310c2 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Actor.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class Actor { + private User user; + private Session session; + @JsonProperty("invoked_by") + private String invokedBy; + private Idp idp; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Api.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Api.java new file mode 100644 index 0000000000..c68dbb4a8f --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Api.java @@ -0,0 +1,12 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Api { + private Response response; + private String operation; + private String version; + private Service service; + private Request request; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Cloud.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Cloud.java new file mode 100644 index 0000000000..e7da0f67d3 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Cloud.java @@ -0,0 +1,9 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Cloud { + private String region; + private String provider; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Endpoint.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Endpoint.java new file mode 100644 index 0000000000..eaa0fde6eb --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Endpoint.java @@ -0,0 +1,10 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Endpoint { + private String uid; + private String ip; + private String domain; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Feature.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Feature.java new file mode 100644 index 0000000000..31ff90bad3 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Feature.java @@ -0,0 +1,8 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Feature { + private String name; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/HttpRequest.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/HttpRequest.java new file mode 100644 index 0000000000..c87f6a059e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/HttpRequest.java @@ -0,0 +1,10 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class HttpRequest { + @JsonProperty("user_agent") + private String userAgent; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Idp.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Idp.java new file mode 100644 index 0000000000..2580bd58b9 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Idp.java @@ -0,0 +1,8 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Idp { + private String name; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Metadata.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Metadata.java new file mode 100644 index 0000000000..efcabeaab4 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Metadata.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +import java.util.List; + +@Data +public class Metadata { + private Product product; + private String uid; + private List profiles; + private String version; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java new file mode 100644 index 0000000000..d7b6baae55 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java @@ -0,0 +1,121 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; + +import java.util.List; +import java.util.Map; + +@Setter +public class OCSF extends DataType { + private Metadata metadata; + private Long time; + private Cloud cloud; + @JsonProperty("dst_endpoint") + private Endpoint dstEndpoint; + @JsonProperty("http_request") + private HttpRequest httpRequest; + @JsonProperty("src_endpoint") + private Endpoint srcEndpoint; + @JsonProperty("class_name") + private String className; + @JsonProperty("class_uid") + private Integer classUid; + @JsonProperty("category_name") + private String categoryName; + @JsonProperty("category_uid") + private Integer categoryUid; + @JsonProperty("severity_id") + private Integer severityId; + private String severity; + private String user; + @JsonProperty("activity_name") + private String activityName; + @JsonProperty("activity_id") + private Integer activityId; + @JsonProperty("type_uid") + private Integer typeUid; + @JsonProperty("type_name") + private String typeName; + private String status; + @JsonProperty("status_id") + private Integer statusId; + private Boolean mfa; + private Api api; + private List resources; + private Actor actor; + private Map unmapped; + + public OCSF() { + super(); + } + + @Override + public Object getValue(final String fieldName) { + switch (fieldName) { + case "metadata.product.version": return metadata.getProduct().getVersion(); + case "metadata.product.name": return metadata.getProduct().getName(); + case "metadata.product.vendor_name": return metadata.getProduct().getVendorName(); + case "metadata.product.feature.name": return metadata.getProduct().getFeature().getName(); + case "metadata.uid": return metadata.getUid(); + case "metadata.profiles": return metadata.getProfiles(); + case "metadata.version": return metadata.getVersion(); + case "time": return time; + case "cloud.region": return cloud.getRegion(); + case "cloud.provider": return cloud.getProvider(); + case "dst_endpoint": return dstEndpoint; + case "http_request.user_agent": return httpRequest.getUserAgent(); + case "src_endpoint.uid": return srcEndpoint.getUid(); + case "src_endpoint.ip": return srcEndpoint.getIp(); + case "src_endpoint.domain": return srcEndpoint.getDomain(); + case "class_name": return className; + case "class_uid": return classUid; + case "category_name": return categoryName; + case "category_uid": return categoryUid; + case "severity_id": return severityId; + case "severity": return severity; + case "user": return user; + case "activity_name": return activityName; + case "activity_id": return activityId; + case "type_uid": return typeUid; + case "type_name": return typeName; + case "status": return status; + case "status_id": return statusId; + case "mfa": return mfa; + case "api.response.error": return api.getResponse().getError(); + case "api.response.message": return api.getResponse().getMessage(); + case "api.operation": return api.getOperation(); + case "api.version": return api.getVersion(); + case "api.service.name": return api.getService().getName(); + case "api.request.uid": return api.getRequest().getUid(); + case "resources": return resources; + case "actor.user.type": return actor.getUser().getType(); + case "actor.user.name": return actor.getUser().getName(); + case "actor.user.uid": return actor.getUser().getUid(); + case "actor.user.uuid": return actor.getUser().getUuid(); + case "actor.user.account_uid": return actor.getUser().getAccountUid(); + case "actor.user.credential_uid": return actor.getUser().getCredentialUid(); + case "actor.session.created_time": return actor.getSession().getCreatedTime(); + case "actor.session.mfa": return actor.getSession().getMfa(); + case "actor.session.issuer": return actor.getSession().getIssuer(); + case "actor.invoked_by": return actor.getInvokedBy(); + case "actor.idp.name": return actor.getIdp().getName(); + case "unmapped": return unmapped; + default: return handleOtherFields(fieldName); + } + } + + private Object handleOtherFields(final String fieldName) { + final String[] parts = fieldName.split("\\."); + if (parts.length == 0 || !"unmapped".equals(parts[0])) { + throw new IllegalArgumentException("Field " + fieldName + " does not exist in class " + getClass().getName()); + } + + if (parts.length == 1) { + return unmapped; + } + + return unmapped.get(parts[1]); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Product.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Product.java new file mode 100644 index 0000000000..c665473cf7 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Product.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class Product { + private String version; + private String name; + @JsonProperty("vendor_name") + private String vendorName; + private Feature feature; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Request.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Request.java new file mode 100644 index 0000000000..336a878145 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Request.java @@ -0,0 +1,8 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Request { + private String uid; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Resource.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Resource.java new file mode 100644 index 0000000000..279597dd74 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Resource.java @@ -0,0 +1,12 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class Resource { + private String uid; + @JsonProperty("account_uid") + private String accountUid; + private String type; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Response.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Response.java new file mode 100644 index 0000000000..7fe3899845 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Response.java @@ -0,0 +1,9 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Response { + private String error; + private String message; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Service.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Service.java new file mode 100644 index 0000000000..9c14617a44 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Service.java @@ -0,0 +1,8 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import lombok.Data; + +@Data +public class Service { + private String name; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Session.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Session.java new file mode 100644 index 0000000000..894b53e9c9 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/Session.java @@ -0,0 +1,12 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class Session { + @JsonProperty("created_time") + private Long createdTime; + private Boolean mfa; + private String issuer; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/User.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/User.java new file mode 100644 index 0000000000..c11030864d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/User.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class User { + private String type; + private String name; + private String uid; + private String uuid; + @JsonProperty("account_uid") + private String accountUid; + @JsonProperty("credential_uid") + private String credentialUid; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf.json b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/ocsf.json similarity index 100% rename from data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf.json rename to data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/ocsf.json diff --git a/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java b/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java index 25ebc0dfb0..0cc898e7e4 100644 --- a/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java +++ b/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java @@ -5,7 +5,7 @@ //import org.mockito.MockitoAnnotations; //import org.opensearch.dataprepper.plugins.processor.model.datatypes.CloudTrail; //import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; -//import org.opensearch.dataprepper.plugins.processor.model.datatypes.OCSF; +//import org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf.OCSF; //import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; // //import java.nio.charset.StandardCharsets; diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index 8f1f721809..b476d15ebb 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:buffer-common') implementation project(':data-prepper-plugins:common') + implementation project(':data-prepper-plugins:rule-engine') implementation libs.armeria.core implementation 'io.micrometer:micrometer-core' implementation 'software.amazon.awssdk:s3' diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodec.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodec.java new file mode 100644 index 0000000000..3fb31b2226 --- /dev/null +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodec.java @@ -0,0 +1,40 @@ +package org.opensearch.dataprepper.plugins.source.s3.codec; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf.OCSF; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.function.Consumer; + +@DataPrepperPlugin(name = "ocsf", pluginType = InputCodec.class, pluginConfigurationType = OCSFCodecConfig.class) +public class OCSFCodec implements InputCodec { + private final ObjectMapper objectMapper = new ObjectMapper(); + + @DataPrepperPluginConstructor + public OCSFCodec(final OCSFCodecConfig config) { + + } + + @Override + public void parse(InputStream inputStream, Consumer> eventConsumer) throws IOException { + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + parseBufferedReader(reader, eventConsumer); + } + } + + private void parseBufferedReader(final BufferedReader reader, final Consumer> eventConsumer) throws IOException { + String line; + while ((line = reader.readLine()) != null) { + final OCSF ocsf = objectMapper.readValue(line, OCSF.class); + eventConsumer.accept(new Record<>(ocsf)); + } + } +} diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodecConfig.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodecConfig.java new file mode 100644 index 0000000000..f4b3e82946 --- /dev/null +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/codec/OCSFCodecConfig.java @@ -0,0 +1,4 @@ +package org.opensearch.dataprepper.plugins.source.s3.codec; + +public class OCSFCodecConfig { +}