diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-enterprise/GeoLite2-Enterprise.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-enterprise/GeoLite2-Enterprise.mmdb new file mode 100644 index 0000000000..837b725e9c Binary files /dev/null and b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-enterprise/GeoLite2-Enterprise.mmdb differ diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb new file mode 100644 index 0000000000..c50fbe9ccc Binary files /dev/null and b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb differ diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb new file mode 100644 index 0000000000..e892807ea6 Binary files /dev/null and b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb differ diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb new file mode 100644 index 0000000000..e59fdad315 Binary files /dev/null and b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb differ diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 13ea63adbe..00814b40c7 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation project(':data-prepper-plugins:buffer-common') implementation project(':data-prepper-plugins:common') implementation project(':data-prepper-plugins:failures-common') + implementation project(':data-prepper-plugins:rule-engine') implementation libs.opensearch.client implementation libs.opensearch.rhlc implementation libs.opensearch.java diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkOperationWrapper.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkOperationWrapper.java index 7f11db2234..3dfb1e4f9f 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkOperationWrapper.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkOperationWrapper.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson; @@ -45,26 +46,32 @@ public class BulkOperationWrapper { private final EventHandle eventHandle; private final BulkOperation bulkOperation; private final SerializedJson jsonNode; + private final Event event; public BulkOperationWrapper(final BulkOperation bulkOperation) { - this(bulkOperation, null, null); + this(bulkOperation, null, null, null); } - public BulkOperationWrapper(final BulkOperation bulkOperation, final EventHandle eventHandle, final SerializedJson jsonNode) { + public BulkOperationWrapper(final BulkOperation bulkOperation, final EventHandle eventHandle, final SerializedJson jsonNode, final Event event) { checkNotNull(bulkOperation); this.bulkOperation = bulkOperation; this.eventHandle = eventHandle; this.jsonNode = jsonNode; + this.event = event; } public BulkOperationWrapper(final BulkOperation bulkOperation, final EventHandle eventHandle) { - this(bulkOperation, eventHandle, null); + this(bulkOperation, eventHandle, null, null); } public BulkOperation getBulkOperation() { return bulkOperation; } + public Event getEvent() { + return event; + } + public EventHandle getEventHandle() { return eventHandle; } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index 2c629e32a6..8f9450c267 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -15,6 +15,7 @@ import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.processor.model.event.EventWrapper; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.rest.RestStatus; @@ -23,6 +24,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -185,21 +187,23 @@ private void incrementErrorCounters(final Exception e) { } } - public void execute(final AccumulatingBulkRequest bulkRequest) throws InterruptedException { + public List execute(final AccumulatingBulkRequest bulkRequest) throws InterruptedException { + final List eventWrappers = new ArrayList<>(); + final Backoff backoff = Backoff.exponential(INITIAL_DELAY_MS, MAXIMUM_DELAY_MS).withMaxAttempts(maxRetries); BulkOperationRequestResponse operationResponse; BulkResponse response = null; AccumulatingBulkRequest request = bulkRequest; int attempt = 1; do { - operationResponse = handleRetry(request, response, attempt); + operationResponse = handleRetry(request, response, attempt, eventWrappers); if (operationResponse != null) { final long delayMillis = backoff.nextDelayMillis(attempt++); request = operationResponse.getBulkRequest(); response = operationResponse.getResponse(); if (delayMillis < 0) { RuntimeException e = new RuntimeException(String.format("Number of retries reached the limit of max retries (configured value %d)", maxRetries)); - handleFailures(request, null, e); + handleFailures(request, null, e, eventWrappers); break; } // Wait for backOff duration @@ -210,6 +214,8 @@ public void execute(final AccumulatingBulkRequest bulkRequest) throws Interrupte } } } while (operationResponse != null); + + return eventWrappers; } public boolean canRetry(final BulkResponse response) { @@ -230,7 +236,8 @@ public static boolean canRetry(final Exception e) { private BulkOperationRequestResponse handleRetriesAndFailures(final AccumulatingBulkRequest bulkRequestForRetry, final int retryCount, final BulkResponse bulkResponse, - final Exception exceptionFromRequest) { + final Exception exceptionFromRequest, + final List eventWrappers) { final boolean doRetry = (Objects.isNull(exceptionFromRequest)) ? canRetry(bulkResponse) : canRetry(exceptionFromRequest); if (!Objects.isNull(bulkResponse) && retryCount == 1) { // first attempt for (final BulkResponseItem bulkItemResponse : bulkResponse.items()) { @@ -253,12 +260,13 @@ private BulkOperationRequestResponse handleRetriesAndFailures(final Accumulating bulkRequestNumberOfRetries.increment(); return new BulkOperationRequestResponse(bulkRequestForRetry, bulkResponse); } else { - handleFailures(bulkRequestForRetry, bulkResponse, exceptionFromRequest); + handleFailures(bulkRequestForRetry, bulkResponse, exceptionFromRequest, eventWrappers); } return null; } - private void handleFailures(final AccumulatingBulkRequest bulkRequest, final BulkResponse bulkResponse, final Throwable failure) { + private void handleFailures(final AccumulatingBulkRequest bulkRequest, final BulkResponse bulkResponse, final Throwable failure, + final List eventWrappers) { if (failure == null) { for (final BulkResponseItem bulkItemResponse : bulkResponse.items()) { // Skip logging the error for version conflicts @@ -266,7 +274,7 @@ private void handleFailures(final AccumulatingBulkRequest bulkRequestForRetry = createBulkRequestForRetry(request, response); + private BulkOperationRequestResponse handleRetry(final AccumulatingBulkRequest request, final BulkResponse response, int retryCount, + final List eventWrappers) throws InterruptedException { + final AccumulatingBulkRequest bulkRequestForRetry = createBulkRequestForRetry(request, response, eventWrappers); if (bulkRequestForRetry.getOperationsCount() == 0) { return null; } @@ -285,10 +294,10 @@ private BulkOperationRequestResponse handleRetry(final AccumulatingBulkRequest r bulkResponse = requestFunction.apply(bulkRequestForRetry); } catch (Exception e) { incrementErrorCounters(e); - return handleRetriesAndFailures(bulkRequestForRetry, retryCount, null, e); + return handleRetriesAndFailures(bulkRequestForRetry, retryCount, null, e, eventWrappers); } if (bulkResponse.errors()) { - return handleRetriesAndFailures(bulkRequestForRetry, retryCount, bulkResponse, null); + return handleRetriesAndFailures(bulkRequestForRetry, retryCount, bulkResponse, null, eventWrappers); } else { final int numberOfDocs = bulkRequestForRetry.getOperationsCount(); final boolean firstAttempt = (retryCount == 1); @@ -296,15 +305,19 @@ private BulkOperationRequestResponse handleRetry(final AccumulatingBulkRequest r sentDocumentsOnFirstAttemptCounter.increment(numberOfDocs); } sentDocumentsCounter.increment(bulkRequestForRetry.getOperationsCount()); - for (final BulkOperationWrapper bulkOperation: bulkRequestForRetry.getOperations()) { + for (int i = 0; i < bulkResponse.items().size(); i++) { + final BulkOperationWrapper bulkOperation = bulkRequestForRetry.getOperationAt(i); + final BulkResponseItem bulkResponseItem = bulkResponse.items().get(i); + bulkOperation.releaseEventHandle(true); + eventWrappers.add(createEventWrapper(bulkOperation, bulkResponseItem)); } } return null; } private AccumulatingBulkRequest createBulkRequestForRetry( - final AccumulatingBulkRequest request, final BulkResponse response) { + final AccumulatingBulkRequest request, final BulkResponse response, final List eventWrappers) { if (response == null) { // first attempt or retry due to Exception return request; @@ -322,6 +335,7 @@ private AccumulatingBulkRequest createBulkReq documentsVersionConflictErrors.increment(); LOG.debug("Received version conflict from OpenSearch: {}", bulkItemResponse.error().reason()); bulkOperation.releaseEventHandle(true); + eventWrappers.add(createEventWrapper(bulkOperation, bulkItemResponse)); } else { nonRetryableFailures.add(FailedBulkOperation.builder() .withBulkOperation(bulkOperation) @@ -332,6 +346,7 @@ private AccumulatingBulkRequest createBulkReq } else { sentDocumentsCounter.increment(); bulkOperation.releaseEventHandle(true); + eventWrappers.add(createEventWrapper(bulkOperation, bulkItemResponse)); } index++; } @@ -343,7 +358,8 @@ private AccumulatingBulkRequest createBulkReq } } - private void handleFailures(final AccumulatingBulkRequest accumulatingBulkRequest, final List itemResponses) { + private void handleFailures(final AccumulatingBulkRequest accumulatingBulkRequest, final List itemResponses, + final List eventWrappers) { assert accumulatingBulkRequest.getOperationsCount() == itemResponses.size(); final ImmutableList.Builder failures = ImmutableList.builder(); for (int i = 0; i < itemResponses.size(); i++) { @@ -354,6 +370,7 @@ private void handleFailures(final AccumulatingBulkRequest> { private DlqProvider dlqProvider; private final ConcurrentHashMap> bulkRequestMap; private final ConcurrentHashMap lastFlushTimeMap; + private RuleEngine ruleEngine = null; @DataPrepperPluginConstructor public OpenSearchSink(final PluginSetting pluginSetting, @@ -261,6 +266,10 @@ private void doInitializeInternal() throws IOException { maybeUpdateServerlessNetworkPolicy(); objectMapper = new ObjectMapper(); + + final Optional ruleEngineConfig = openSearchSinkConfig.getRuleEngineConfig(); + ruleEngineConfig.ifPresent(engineConfig -> ruleEngine = new RuleEngine(engineConfig, openSearchClient)); + this.initialized = true; LOG.info("Initialized OpenSearch sink"); } @@ -440,7 +449,7 @@ public void doOutput(final Collection> records) { continue; } - BulkOperationWrapper bulkOperationWrapper = new BulkOperationWrapper(bulkOperation, event.getEventHandle(), serializedJsonNode); + BulkOperationWrapper bulkOperationWrapper = new BulkOperationWrapper(bulkOperation, event.getEventHandle(), serializedJsonNode, event); final long estimatedBytesBeforeAdd = bulkRequest.estimateSizeInBytesWithDocument(bulkOperationWrapper); if (bulkSize >= 0 && estimatedBytesBeforeAdd >= bulkSize && bulkRequest.getOperationsCount() > 0) { flushBatch(bulkRequest); @@ -494,7 +503,10 @@ private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { bulkRequestTimer.record(() -> { try { LOG.debug("Sending data to OpenSearch"); - bulkRetryStrategy.execute(accumulatingBulkRequest); + final List eventWrappers = bulkRetryStrategy.execute(accumulatingBulkRequest); + if (ruleEngine != null) { + ruleEngine.doExecute(eventWrappers); + } bulkRequestSizeBytesSummary.record(accumulatingBulkRequest.getEstimatedSizeInBytes()); } catch (final InterruptedException e) { LOG.error("Unexpected Interrupt:", e); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 4ec8f17bc8..d71935f371 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -6,9 +6,13 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfig; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfigWrapper; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import java.util.Optional; + import static com.google.common.base.Preconditions.checkNotNull; public class OpenSearchSinkConfiguration { @@ -16,6 +20,7 @@ public class OpenSearchSinkConfiguration { private final ConnectionConfiguration connectionConfiguration; private final IndexConfiguration indexConfiguration; private final RetryConfiguration retryConfiguration; + private final RuleEngineConfigWrapper ruleEngineConfigWrapper; public ConnectionConfiguration getConnectionConfiguration() { return connectionConfiguration; @@ -29,15 +34,21 @@ public RetryConfiguration getRetryConfiguration() { return retryConfiguration; } + public Optional getRuleEngineConfig() { + return Optional.ofNullable(ruleEngineConfigWrapper.getRuleEngineConfig()); + } + private OpenSearchSinkConfiguration( final ConnectionConfiguration connectionConfiguration, final IndexConfiguration indexConfiguration, - final RetryConfiguration retryConfiguration) { + final RetryConfiguration retryConfiguration, final RuleEngineConfigWrapper ruleEngineConfigWrapper) { checkNotNull(connectionConfiguration, "connectionConfiguration cannot be null"); checkNotNull(indexConfiguration, "indexConfiguration cannot be null"); checkNotNull(retryConfiguration, "retryConfiguration cannot be null"); + checkNotNull(ruleEngineConfigWrapper, "ruleEngineConfigWrapper cannot be null"); this.connectionConfiguration = connectionConfiguration; this.indexConfiguration = indexConfiguration; this.retryConfiguration = retryConfiguration; + this.ruleEngineConfigWrapper = ruleEngineConfigWrapper; } public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting) { @@ -49,7 +60,8 @@ public static OpenSearchSinkConfiguration readESConfig(final PluginSetting plugi ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(pluginSetting); + final RuleEngineConfigWrapper ruleEngineConfigWrapper = RuleEngineConfigWrapper.readRuleEngineConfigWrapper(pluginSetting); - return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); + return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration, ruleEngineConfigWrapper); } } diff --git a/data-prepper-plugins/rule-engine/build.gradle b/data-prepper-plugins/rule-engine/build.gradle new file mode 100644 index 0000000000..b36dc360fd --- /dev/null +++ b/data-prepper-plugins/rule-engine/build.gradle @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java' + id 'antlr' + id 'idea' +} + +ext { + antlrGeneratedPackageDirectory = "org/opensearch/dataprepper/plugins/processor/rules/antlr/" +} + +dependencies { + antlr('org.antlr:antlr4:4.10.1') { + exclude group: 'org.glassfish', module: 'javax.json' + } + implementation project(':data-prepper-api') + implementation 'javax.inject:javax.inject:1' + implementation(libs.spring.core) { + exclude group: 'commons-logging', module: 'commons-logging' + } + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } + implementation platform('org.apache.logging.log4j:log4j-bom:2.22.1') + implementation 'org.apache.logging.log4j:log4j-core' + implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' + implementation 'com.github.seancfoley:ipaddress:5.4.2' + implementation libs.opensearch.client + implementation libs.opensearch.rhlc + implementation libs.opensearch.java + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation libs.commons.lang3 +} + +generateGrammarSource { + outputDirectory = new File("build/generated-src/antlr/main/${antlrGeneratedPackageDirectory}") + arguments += ['-package', 'org.opensearch.dataprepper.plugins.processor.rules.antlr'] + arguments += ['-visitor'] +} + +jacocoTestCoverageVerification { + dependsOn jacocoTestReport + violationRules { + rule { //in addition to core projects rule - this one checks for 100% code coverage for this project + limit { + minimum = 1.0 // keep at 100% + } + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: ["${antlrGeneratedPackageDirectory}/**"]) + })) + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/antlr/Condition.g4 b/data-prepper-plugins/rule-engine/src/main/antlr/Condition.g4 new file mode 100644 index 0000000000..c4aba35ca1 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/antlr/Condition.g4 @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +grammar Condition; + +AND: 'and' ; +OR: 'or' ; +NOT: 'not'; + +LPAREN : '(' ; +RPAREN : ')' ; + +SELECTOR: ('1'|'any'|'all')( ' of ' )[A-Za-z_0-9*]+ ; +IDENTIFIER: [a-zA-Z_] [a-zA-Z_0-9]* ; +WHITESPACE: [ \r\n\t]+ -> skip; + +start : expression; + +expression + : (SELECTOR|IDENTIFIER) # identOrSelectExpression + | LPAREN inner=expression RPAREN # parenExpression + | NOT expression # notExpression + | left=expression operator=AND right=expression # andExpression + | left=expression operator=OR right=expression # orExpression + ; \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java new file mode 100644 index 0000000000..7de01b3667 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java @@ -0,0 +1,131 @@ +package org.opensearch.dataprepper.plugins.processor; + +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.dataprepper.plugins.processor.generator.FindingGenerator; +import org.opensearch.dataprepper.plugins.processor.model.event.EventWrapper; +import org.opensearch.dataprepper.plugins.processor.model.findings.Finding; +import org.opensearch.dataprepper.plugins.processor.rules.Rule; +import org.opensearch.dataprepper.plugins.processor.rules.RuleConverter; +import org.opensearch.dataprepper.plugins.processor.rules.RuleFetcher; +import org.opensearch.dataprepper.plugins.processor.rules.RuleStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class RuleEngine { + private static final String FINDINGS_INDEX = ".opensearch-sap-cloudtrail-findings"; + + private static final Logger LOG = LoggerFactory.getLogger(RuleEngine.class); + + private static final ScheduledExecutorService SCHEDULED_EXECUTOR_SERVICE = Executors.newSingleThreadScheduledExecutor(); + + private final RuleStore ruleStore; + private final FindingGenerator findingGenerator; + private final OpenSearchClient openSearchClient; + + public RuleEngine(final RuleEngineConfig config, final OpenSearchClient openSearchClient) { + ruleStore = new RuleStore(); + findingGenerator = new FindingGenerator(); + this.openSearchClient = openSearchClient; + setupRuleFetching(config, ruleStore); + } + + public void doExecute(final Collection eventWrappers) { + if (eventWrappers.isEmpty()) { + return; + } + + final Map> indexToSigmaRules = ruleStore.getRules(); + LOG.info("Got index to rules: {}", indexToSigmaRules); + + final Map> allFindings = new HashMap<>(); + eventWrappers.forEach(eventWrapper -> { + final String indexName = eventWrapper.getIndexName(); + final List sigmaRules = indexToSigmaRules.get(indexName); + if (sigmaRules == null || sigmaRules.isEmpty()) { + return; + } + + // TODO - should just model after OpenSearch SAP, have list of detectors, iterate over them + final List ruleMatches = sigmaRules.stream() + .filter(sigmaRule -> sigmaRule.getCondition().test(eventWrapper.getEvent())) + .collect(Collectors.toList()); + + if (!ruleMatches.isEmpty()) { + final Map> findingsIndexToFindings = findingGenerator.generateFindings(eventWrapper, ruleMatches); + findingsIndexToFindings.forEach((findingsIndex, findings) -> { + allFindings.putIfAbsent(findingsIndex, new ArrayList<>()); + allFindings.get(findingsIndex).addAll(findings); + }); + } + }); + + indexFindings(allFindings); + } + + private void indexFindings(final Map> allFindings) { + if (allFindings.isEmpty()) { + LOG.debug("No findings to index"); + return; + } + + final BulkRequest bulkRequest = createBulkRequest(allFindings); + try { + final BulkResponse bulkResponse = openSearchClient.bulk(bulkRequest); + if (bulkResponse.errors()) { + LOG.error("BulkResponse has errors"); + bulkResponse.items().stream() + .filter(bulkResponseItem -> bulkResponseItem.error() != null) + .forEach(bulkResponseItem -> LOG.error("BulkItemError for ID {}: {}", bulkResponseItem.id(), bulkResponseItem.error().reason())); + } + } catch (IOException e) { + LOG.error("Caught exception indexing findings", e); + } + } + + private BulkRequest createBulkRequest(final Map> allFindings) { + final List allBulkOperations = new ArrayList<>(); + allFindings.forEach((findingsIndex, findings) -> { + final List bulkOperations = findings.stream() + .peek(finding -> LOG.info("Indexing finding with ID: {}", finding.getId())) + .map(finding -> new IndexOperation.Builder<>() + .id(finding.getId()) + .index(findingsIndex) + .document(finding) + .build()) + .map(idxOp -> new BulkOperation.Builder().index(idxOp).build()) + .collect(Collectors.toList()); + allBulkOperations.addAll(bulkOperations); + }); + + return new BulkRequest.Builder() + .operations(allBulkOperations) + .build(); + } + + private void setupRuleFetching(final RuleEngineConfig config, final RuleStore ruleStore) { + final RuleConverter ruleConverter = new RuleConverter(config); + final RuleFetcher ruleFetcher = new RuleFetcher(openSearchClient, ruleStore, ruleConverter); + SCHEDULED_EXECUTOR_SERVICE.scheduleAtFixedRate( + ruleFetcher, + 0L, + config.getRuleRefreshInterval().toMillis(), + TimeUnit.MILLISECONDS + ); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java new file mode 100644 index 0000000000..b2c106340c --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java @@ -0,0 +1,85 @@ +package org.opensearch.dataprepper.plugins.processor; + +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.processor.model.log.LogFormat; +import org.opensearch.dataprepper.plugins.processor.model.log.LogType; + +import java.time.Duration; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class RuleEngineConfig { + static final long DEFAULT_RULE_REFRESH_INTERVAL_MILLIS = Duration.ofMinutes(1).toMillis(); + static final LogFormat DEFAULT_LOG_FORMAT = LogFormat.NONE; + + public static final String RULE_REFRESH_INTERVAL_MILLIS = "rule_refresh_interval_millis"; + public static final String LOG_TYPE = "log_type"; + public static final String LOG_FORMAT = "log_format"; + + private final Duration ruleRefreshInterval; + private final LogType logType; + private final LogFormat logFormat; + + public Duration getRuleRefreshInterval() { + return ruleRefreshInterval; + } + + public LogType getLogType() { + return logType; + } + + public LogFormat getLogFormat() { + return logFormat; + } + + public static class Builder { + private Duration ruleRefreshInterval = Duration.ofMillis(DEFAULT_RULE_REFRESH_INTERVAL_MILLIS); + private LogType logType; + private LogFormat logFormat = DEFAULT_LOG_FORMAT; + + public Builder withRuleRefreshInterval(final Duration ruleRefreshInterval) { + checkNotNull(ruleRefreshInterval, "ruleRefreshInterval cannot be null."); + this.ruleRefreshInterval = ruleRefreshInterval; + return this; + } + + public Builder withLogType(final LogType logType) { + checkNotNull(logType, "logType cannot be null."); + this.logType = logType; + return this; + } + + public Builder withLogFormat(final LogFormat logFormat) { + checkNotNull(logFormat, "logFormat cannot be null"); + this.logFormat = logFormat; + return this; + } + + public RuleEngineConfig build() { + return new RuleEngineConfig(this); + } + } + + private RuleEngineConfig(final Builder builder) { + this.ruleRefreshInterval = builder.ruleRefreshInterval; + this.logType = checkNotNull(builder.logType, "logType cannot be null."); + this.logFormat = builder.logFormat; + } + + public static RuleEngineConfig readRuleEngineConfig(final PluginSetting pluginSetting) { + RuleEngineConfig.Builder builder = new RuleEngineConfig.Builder(); + + final Duration ruleRefreshInterval = Duration.ofMillis( + pluginSetting.getLongOrDefault(RULE_REFRESH_INTERVAL_MILLIS, DEFAULT_RULE_REFRESH_INTERVAL_MILLIS) + ); + builder = builder.withRuleRefreshInterval(ruleRefreshInterval); + + final LogType logType = LogType.valueOf(pluginSetting.getStringOrDefault(LOG_TYPE, null).toUpperCase()); + builder = builder.withLogType(logType); + + final LogFormat logFormat = LogFormat.valueOf(pluginSetting.getStringOrDefault(LOG_FORMAT, DEFAULT_LOG_FORMAT.name()).toUpperCase()); + builder = builder.withLogFormat(logFormat); + + return builder.build(); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfigWrapper.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfigWrapper.java new file mode 100644 index 0000000000..35999fa8c5 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfigWrapper.java @@ -0,0 +1,49 @@ +package org.opensearch.dataprepper.plugins.processor; + +import org.opensearch.dataprepper.model.configuration.PluginSetting; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class RuleEngineConfigWrapper { + public static final String RULE_ENGINE = "rule_engine"; + + private final RuleEngineConfig ruleEngineConfig; + + public RuleEngineConfig getRuleEngineConfig() { + return ruleEngineConfig; + } + + public static class Builder { + private RuleEngineConfig ruleEngineConfig; + + public Builder withRuleEngineConfig(final RuleEngineConfig ruleEngineConfig) { + checkNotNull(ruleEngineConfig, "ruleEngineConfig cannot be null."); + this.ruleEngineConfig = ruleEngineConfig; + return this; + } + + public RuleEngineConfigWrapper build() { + return new RuleEngineConfigWrapper(this); + } + } + + private RuleEngineConfigWrapper(final Builder builder) { + this.ruleEngineConfig = builder.ruleEngineConfig; + } + + public static RuleEngineConfigWrapper readRuleEngineConfigWrapper(final PluginSetting pluginSetting) { + RuleEngineConfigWrapper.Builder builder = new RuleEngineConfigWrapper.Builder(); + + final Map ruleEngineSettingsMap = (Map) pluginSetting.getAttributeFromSettings(RULE_ENGINE); + if (ruleEngineSettingsMap != null) { + final PluginSetting ruleEngineSettings = new PluginSetting(RULE_ENGINE, ruleEngineSettingsMap); + final RuleEngineConfig ruleEngineConfig = RuleEngineConfig.readRuleEngineConfig(ruleEngineSettings); + builder = builder.withRuleEngineConfig(ruleEngineConfig); + } + + return builder.build(); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/MappingException.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/MappingException.java new file mode 100644 index 0000000000..323807e13b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/MappingException.java @@ -0,0 +1,7 @@ +package org.opensearch.dataprepper.plugins.processor.exceptions; + +public class MappingException extends RuntimeException { + public MappingException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/RuleRefreshException.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/RuleRefreshException.java new file mode 100644 index 0000000000..991252533d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/exceptions/RuleRefreshException.java @@ -0,0 +1,7 @@ +package org.opensearch.dataprepper.plugins.processor.exceptions; + +public class RuleRefreshException extends RuntimeException { + public RuleRefreshException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java new file mode 100644 index 0000000000..c0f156b927 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java @@ -0,0 +1,70 @@ +package org.opensearch.dataprepper.plugins.processor.formats.accessors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfig; +import org.opensearch.dataprepper.plugins.processor.mappings.MappingParser; +import org.opensearch.dataprepper.plugins.processor.model.log.LogFormat; +import org.opensearch.dataprepper.plugins.processor.model.log.LogType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class FieldAccessor { + private static final Logger LOG = LoggerFactory.getLogger(FieldAccessor.class); + + private final Map mappings; + + public FieldAccessor(final RuleEngineConfig config) { + final ObjectMapper objectMapper = new ObjectMapper(); + final MappingParser mappingParser = new MappingParser(objectMapper); + + final LogType logType = config.getLogType(); + LOG.info("Log type: {}", logType.name()); + final LogFormat logFormat = config.getLogFormat(); + LOG.info("Log format: {}", logFormat.name()); + mappings = mappingParser.parseMappings(logType, logFormat); + } + + public String getStringValue(final Event event, final String fieldName) { + return getValue(event, fieldName, String.class); + } + + public Boolean getBooleanValue(final Event event, final String fieldName) { + return getValue(event, fieldName, Boolean.class); + } + + public Integer getIntegerValue(final Event event, final String fieldName) { + return getValue(event, fieldName, Integer.class); + } + + public Float getFloatValue(final Event event, final String fieldName) { + return getValue(event, fieldName, Float.class); + } + + public Object getObjectValue(final Event event, final String fieldName) { + return getValue(event, fieldName, Object.class); + } + + private T getValue(final Event event, final String fieldName, final Class clazz) { + //LOG.info("Field: {}", fieldName); + final String mappedField = convertFieldName(fieldName); + //System.out.println("Mapped field: " + mappedField); + final String jsonPointer = getJsonPointer(mappedField); + //System.out.println("Pointer: "+ jsonPointer); + final T value = event.get(jsonPointer, clazz); + //LOG.info("Value: {}", value); + return value; + } + + private String convertFieldName(final String fieldName) { + return mappings.get(fieldName); + } + + // TODO - need flag for this + private String getJsonPointer(final String field) { + //return field; + return field.replace(".", "/"); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/generator/FindingGenerator.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/generator/FindingGenerator.java new file mode 100644 index 0000000000..6a0c5ceb66 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/generator/FindingGenerator.java @@ -0,0 +1,54 @@ +package org.opensearch.dataprepper.plugins.processor.generator; + +import org.opensearch.dataprepper.plugins.processor.model.event.EventWrapper; +import org.opensearch.dataprepper.plugins.processor.model.findings.DocLevelQuery; +import org.opensearch.dataprepper.plugins.processor.model.findings.Finding; +import org.opensearch.dataprepper.plugins.processor.rules.Rule; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +public class FindingGenerator { + public Map> generateFindings(final EventWrapper eventWrapper, final List ruleMatches) { + final Map> monitorIdToRules = new HashMap<>(); + ruleMatches.forEach(rule -> { + final String monitorId = rule.getMonitorId(); + monitorIdToRules.putIfAbsent(monitorId, new ArrayList<>()); + monitorIdToRules.get(monitorId).add(rule); + }); + + final Map> findingsIndexToFindings = new HashMap<>(); + monitorIdToRules.forEach((monitorId, rules) -> { + final String findingsIndex = rules.get(0).getFindingsIndex(); + + final List queries = rules.stream() + .map(this::createDocLevelQuery) + .collect(Collectors.toList()); + + final Finding finding = new Finding( + UUID.randomUUID().toString(), + List.of(eventWrapper.getDocId()), + List.of(eventWrapper.getDocId()), + monitorId, + rules.get(0).getMonitorName(), + eventWrapper.getIndexName(), + queries, + Instant.now().toEpochMilli() + ); + + findingsIndexToFindings.putIfAbsent(findingsIndex, new ArrayList<>()); + findingsIndexToFindings.get(findingsIndex).add(finding); + }); + + return findingsIndexToFindings; + } + + private DocLevelQuery createDocLevelQuery(final Rule rule) { + return new DocLevelQuery(rule.getId(), rule.getId(), rule.getTags(), rule.getQuery()); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/mappings/MappingParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/mappings/MappingParser.java new file mode 100644 index 0000000000..05604cca22 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/mappings/MappingParser.java @@ -0,0 +1,40 @@ +package org.opensearch.dataprepper.plugins.processor.mappings; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.plugins.processor.exceptions.MappingException; +import org.opensearch.dataprepper.plugins.processor.model.log.LogFormat; +import org.opensearch.dataprepper.plugins.processor.model.log.LogType; +import org.opensearch.dataprepper.plugins.processor.model.mappings.Mapping; + +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.stream.Collectors; + +public class MappingParser { + private static final String MAPPING_PATH_FORMAT = "mappings/%s"; + + private final ObjectMapper objectMapper; + + public MappingParser(final ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public Map parseMappings(final LogType logType, final LogFormat logFormat) { + final Mapping mapping = getMapping(logType); + + return mapping.getMappings().stream() + .map(fieldMapping -> Map.entry(fieldMapping.get(LogFormat.NONE.getKeyName()), fieldMapping.get(logFormat.getKeyName()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Mapping getMapping(final LogType logType) { + final String relativePath = String.format(MAPPING_PATH_FORMAT, logType.getMappingsFile()); + final URL mappingsPath = getClass().getClassLoader().getResource(relativePath); + try { + return objectMapper.readValue(mappingsPath, Mapping.class); + } catch (final IOException e) { + throw new MappingException("Exception reading mapping: " + relativePath, e); + } + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Detector.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Detector.java new file mode 100644 index 0000000000..b01ff9fe16 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Detector.java @@ -0,0 +1,48 @@ +package org.opensearch.dataprepper.plugins.processor.model.detector; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class Detector { + private String name; + private List inputs; + @JsonProperty("findings_index") + private String findingsIndex; + @JsonProperty("monitor_id") + private List monitorId; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public List getInputs() { + return inputs; + } + + public void setInputs(final List inputs) { + this.inputs = inputs; + } + + public String getFindingsIndex() { + return findingsIndex; + } + + public void setFindingsIndex(final String findingsIndex) { + this.findingsIndex = findingsIndex; + } + + public List getMonitorId() { + return monitorId; + } + + public void setMonitorId(final List monitorId) { + this.monitorId = monitorId; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorInput.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorInput.java new file mode 100644 index 0000000000..190e5901cc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorInput.java @@ -0,0 +1,39 @@ +package org.opensearch.dataprepper.plugins.processor.model.detector; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class DetectorInput { + private List indices; + @JsonProperty("custom_rules") + private List customRules; + @JsonProperty("pre_packaged_rules") + private List prePackagedRules; + + public List getIndices() { + return indices; + } + + public void setIndices(final List indices) { + this.indices = indices; + } + + public List getCustomRules() { + return customRules; + } + + public void setCustomRules(final List customRules) { + this.customRules = customRules; + } + + public List getPrePackagedRules() { + return prePackagedRules; + } + + public void setPrePackagedRules(final List prePackagedRules) { + this.prePackagedRules = prePackagedRules; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorRule.java new file mode 100644 index 0000000000..f812417dcf --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorRule.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.processor.model.detector; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class DetectorRule { + private String id; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorWrapper.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorWrapper.java new file mode 100644 index 0000000000..763cb629e4 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/DetectorWrapper.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.processor.model.detector; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class DetectorWrapper { + private Detector detector; + + public Detector getDetector() { + return detector; + } + + public void setDetector(final Detector detector) { + this.detector = detector; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Input.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Input.java new file mode 100644 index 0000000000..605869a3a5 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/detector/Input.java @@ -0,0 +1,18 @@ +package org.opensearch.dataprepper.plugins.processor.model.detector; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class Input { + @JsonProperty("detector_input") + private DetectorInput detectorInput; + + public DetectorInput getDetectorInput() { + return detectorInput; + } + + public void setDetectorInput(DetectorInput detectorInput) { + this.detectorInput = detectorInput; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/event/EventWrapper.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/event/EventWrapper.java new file mode 100644 index 0000000000..c00df2a72d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/event/EventWrapper.java @@ -0,0 +1,27 @@ +package org.opensearch.dataprepper.plugins.processor.model.event; + +import org.opensearch.dataprepper.model.event.Event; + +public class EventWrapper { + private final String indexName; + private final String docId; + private final Event event; + + public EventWrapper(final String indexName, final String docId, final Event event) { + this.indexName = indexName; + this.docId = docId; + this.event = event; + } + + public String getIndexName() { + return indexName; + } + + public String getDocId() { + return docId; + } + + public Event getEvent() { + return event; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/DocLevelQuery.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/DocLevelQuery.java new file mode 100644 index 0000000000..83373f3917 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/DocLevelQuery.java @@ -0,0 +1,33 @@ +package org.opensearch.dataprepper.plugins.processor.model.findings; + +import java.util.List; + +public class DocLevelQuery { + private String id; + private String name; + private List tags; + private String query; + + public DocLevelQuery(final String id, final String name, final List tags, final String query) { + this.id = id; + this.name = name; + this.tags = tags; + this.query = query; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public List getTags() { + return tags; + } + + public String getQuery() { + return query; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/Finding.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/Finding.java new file mode 100644 index 0000000000..8fcce3db0d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/findings/Finding.java @@ -0,0 +1,64 @@ +package org.opensearch.dataprepper.plugins.processor.model.findings; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public class Finding { + private String id; + @JsonProperty("related_doc_ids") + private List relatedDocIds; + @JsonProperty("correlated_doc_ids") + private List correlatedDocIds; + @JsonProperty("monitor_id") + private String monitorId; + @JsonProperty("monitor_name") + private String monitorName; + private String index; + private List queries; + private long timestamp; + + public Finding(final String id, final List relatedDocIds, final List correlatedDocIds, final String monitorId, + final String monitorName, final String index, final List queries, final long timestamp) { + this.id = id; + this.relatedDocIds = relatedDocIds; + this.correlatedDocIds = correlatedDocIds; + this.monitorId = monitorId; + this.monitorName = monitorName; + this.index = index; + this.queries = queries; + this.timestamp = timestamp; + } + + public String getId() { + return id; + } + + public List getRelatedDocIds() { + return relatedDocIds; + } + + public List getCorrelatedDocIds() { + return correlatedDocIds; + } + + public String getMonitorId() { + return monitorId; + } + + public String getMonitorName() { + return monitorName; + } + + public String getIndex() { + return index; + } + + public List getQueries() { + return queries; + } + + public long getTimestamp() { + return timestamp; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogFormat.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogFormat.java new file mode 100644 index 0000000000..ac15de8825 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogFormat.java @@ -0,0 +1,17 @@ +package org.opensearch.dataprepper.plugins.processor.model.log; + +public enum LogFormat { + NONE("raw_field"), + OCSF("ocsf"), + ECS("ecs"); + + private final String keyName; + + LogFormat(final String keyName) { + this.keyName = keyName; + } + + public String getKeyName() { + return keyName; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogType.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogType.java new file mode 100644 index 0000000000..7e50d8e607 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/log/LogType.java @@ -0,0 +1,15 @@ +package org.opensearch.dataprepper.plugins.processor.model.log; + +public enum LogType { + CLOUDTRAIL("cloudtrail.json"); + + private final String mappingsFile; + + LogType(final String mappingsFile) { + this.mappingsFile = mappingsFile; + } + + public String getMappingsFile() { + return mappingsFile; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/mappings/Mapping.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/mappings/Mapping.java new file mode 100644 index 0000000000..da5617685f --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/mappings/Mapping.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.processor.model.mappings; + +import java.util.List; +import java.util.Map; + +public class Mapping { + private List> mappings; + + public List> getMappings() { + return mappings; + } + + public void setMappings(final List> mappings) { + this.mappings = mappings; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/Rule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/Rule.java new file mode 100644 index 0000000000..2ba6beed8d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/Rule.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.processor.model.rule; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown=true) +public class Rule { + private String rule; + + public String getRule() { + return rule; + } + + public void setRule(String rule) { + this.rule = rule; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleWrapper.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleWrapper.java new file mode 100644 index 0000000000..05da5a11e9 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleWrapper.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.model.rule; + +public class RuleWrapper { + private Rule rule; + + public Rule getRule() { + return rule; + } + + public void setRule(Rule rule) { + this.rule = rule; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRuleParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRuleParser.java new file mode 100644 index 0000000000..2891ecb7a6 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRuleParser.java @@ -0,0 +1,7 @@ +package org.opensearch.dataprepper.plugins.processor.parser; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; + +public interface SigmaRuleParser { + T parseRule(SigmaRule sigmaRule); +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParser.java new file mode 100644 index 0000000000..a1827adf47 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParser.java @@ -0,0 +1,143 @@ +package org.opensearch.dataprepper.plugins.processor.parser; + +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfig; +import org.opensearch.dataprepper.plugins.processor.formats.accessors.FieldAccessor; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionAND; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionFieldEqualsValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionNOT; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionOR; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaCondition; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaBool; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaFloat; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaInteger; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaNull; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Predicate; +import java.util.stream.Stream; + +public class SigmaRulePredicateParser implements SigmaRuleParser> { + private static final Logger LOG = LoggerFactory.getLogger(SigmaRulePredicateParser.class); + + private final FieldAccessor fieldAccessor; + + public SigmaRulePredicateParser(final RuleEngineConfig config) { + this.fieldAccessor = new FieldAccessor(config); + } + + @Override + public Predicate parseRule(final SigmaRule sigmaRule) { + return sigmaRule.getDetection().getParsedConditions().stream() + .map(SigmaCondition::parsed) + .map(this::parsePredicateFromConditionItem) + // TODO - Not sure on this, need to figure out how there could be multiple conditions for same rule + .reduce(Predicate::and) + // Default to no match if there were no predicates + .orElse(x -> false); + } + + private Predicate parsePredicateFromConditionItem(final ConditionItem conditionItem) { + if (conditionItem instanceof ConditionAND) { + return convertAndCondition(conditionItem); + } else if (conditionItem instanceof ConditionOR) { + return convertOrCondition(conditionItem); + } else if (conditionItem instanceof ConditionNOT) { + return convertNotCondition(conditionItem); + } else if (conditionItem instanceof ConditionFieldEqualsValueExpression) { + return convertFieldEquals((ConditionFieldEqualsValueExpression) conditionItem); + } else if (conditionItem instanceof ConditionValueExpression) { + // TODO - not sure what this means yet + throw new UnsupportedOperationException("Can't proceed from ConditionValueExpression yet"); + } else { + throw new IllegalArgumentException("Unexpected condition type class in condition parse tree: " + conditionItem.getClass().getName()); + } + } + + private Predicate convertAndCondition(final ConditionItem condition) { + return getPredicatesFromConditions(condition) + .reduce(Predicate::and) + // TODO - not sure on this, need to figure out why right would be a string to land here with an empty optional + .orElse(x -> true); + } + + private Predicate convertOrCondition(final ConditionItem condition) { + return getPredicatesFromConditions(condition) + .reduce(Predicate::or) + // TODO - not sure on this, need to figure out why right would be a string to land here with an empty optional + .orElse(x -> true); + } + + private Predicate convertNotCondition(final ConditionItem condition) { + return getPredicatesFromConditions(condition) + .map(Predicate::negate) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Expected exactly on condition for NOT operator")); + } + + private Stream> getPredicatesFromConditions(final ConditionItem condition) { + return condition.getArgs().stream() + // Filter on is another condition + .filter(Either::isLeft) + .map(Either::getLeft) + .map(this::parsePredicateFromConditionItem); + } + + private Predicate convertFieldEquals(final ConditionFieldEqualsValueExpression condition) { + final SigmaType conditionValue = condition.getValue(); + + if (conditionValue instanceof SigmaString) { + return convertStringEquals(condition); + } else if (conditionValue instanceof SigmaBool) { + return convertBooleanEquals(condition); + } else if (conditionValue instanceof SigmaInteger) { + return convertIntegerEquals(condition); + } else if (conditionValue instanceof SigmaFloat) { + return convertFloatEquals(condition); + } else if (conditionValue instanceof SigmaNull) { + return convertNullEquals(condition); + } else { + throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + conditionValue.getClass().getName()); + } + } + + private Predicate convertStringEquals(final ConditionFieldEqualsValueExpression condition) { + final SigmaString sigmaString = (SigmaString) condition.getValue(); + return event -> { + //LOG.info("Field: {}", condition.getField()); + //LOG.info("Expected value: {}", sigmaString.getOriginal()); + //LOG.info("Event: {}", event.toJsonString()); + + final String value = fieldAccessor.getStringValue(event, condition.getField()); + //LOG.info("Value: {}", value); + + return sigmaString.getOriginal().equals(value); + }; + } + + private Predicate convertBooleanEquals(final ConditionFieldEqualsValueExpression condition) { + final SigmaBool sigmaBool = (SigmaBool) condition.getValue(); + return event -> sigmaBool.getBoolean().equals(fieldAccessor.getBooleanValue(event, condition.getField())); + } + + private Predicate convertIntegerEquals(final ConditionFieldEqualsValueExpression condition) { + final SigmaInteger sigmaInteger = (SigmaInteger) condition.getValue(); + return event -> sigmaInteger.getInteger().equals(fieldAccessor.getIntegerValue(event, condition.getField())); + } + + private Predicate convertFloatEquals(final ConditionFieldEqualsValueExpression condition) { + final SigmaFloat sigmaFloat = (SigmaFloat) condition.getValue(); + return event -> sigmaFloat.getFloat().equals(fieldAccessor.getFloatValue(event, condition.getField())); + } + + private Predicate convertNullEquals(final ConditionFieldEqualsValueExpression condition) { + return event -> fieldAccessor.getObjectValue(event, condition.getField()) == null; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionAND.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionAND.java new file mode 100644 index 0000000000..c1a89db7b0 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionAND.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class ConditionAND extends ConditionItem { + + private int argCount; + private boolean operator; + + public ConditionAND(boolean tokenList, List> args) { + super(2, tokenList, args); + this.argCount = 2; + this.operator = true; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionFieldEqualsValueExpression.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionFieldEqualsValueExpression.java new file mode 100644 index 0000000000..b8cc594e93 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionFieldEqualsValueExpression.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetections; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.Collections; + +public class ConditionFieldEqualsValueExpression extends ConditionItem { + + private String field; + private SigmaType value; + + private Either parent; + + public ConditionFieldEqualsValueExpression(String field, SigmaType value) { + super(2, false, Collections.emptyList()); + this.field = field; + this.value = value; + } + + public ConditionFieldEqualsValueExpression postProcess(SigmaDetections detections, Object parent) { + this.parent = parent instanceof ConditionItem? Either.left((ConditionItem) parent): Either.right((SigmaDetectionItem) parent); + return this; + } + + public String getField() { + return field; + } + + public SigmaType getValue() { + return value; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionIdentifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionIdentifier.java new file mode 100644 index 0000000000..904be6f965 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionIdentifier.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetection; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetections; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class ConditionIdentifier extends ConditionItem { + + private int argCount; + private boolean tokenList; + private String identifier; + + public ConditionIdentifier(List> args) { + super(1, true, args); + this.argCount = 1; + this.tokenList = true; + this.identifier = args.get(0).get(); + } + + public ConditionItem postProcess(SigmaDetections detections, Object parent) throws SigmaConditionError { + this.setParent((ConditionItem) parent); + + if (detections.getDetections().containsKey(this.identifier)) { + SigmaDetection detection = detections.getDetections().get(this.identifier); + return detection.postProcess(detections, this); + } else { + throw new SigmaConditionError("Detection '" + this.identifier + "' not defined in detections"); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionItem.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionItem.java new file mode 100644 index 0000000000..2c22a4a03c --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionItem.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetections; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.List; + +public class ConditionItem { + + private int argCount; + private boolean tokenList; + private List> args; + + private Either parent; + private boolean operator; + + public ConditionItem(int argCount, boolean tokenList, + List> args) { + this.argCount = argCount; + this.tokenList = tokenList; + this.args = args; + } + + public ConditionItem postProcess(SigmaDetections detections, Object parent) throws SigmaConditionError { + this.parent = parent instanceof ConditionItem? Either.left((ConditionItem) parent): Either.right((SigmaDetectionItem) parent); + + List> newArgs = new ArrayList<>(); + for (Either arg: this.args) { + newArgs.add(Either.left(arg.getLeft().postProcess(detections, parent))); + } + this.args = newArgs; + return this; + } + + public void setParent(ConditionItem parent) { + this.parent = Either.left(parent); + } + + public List> getArgs() { + return args; + } + + public void setArgs(List> args) { + this.args = args; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionNOT.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionNOT.java new file mode 100644 index 0000000000..cdf3affbb4 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionNOT.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class ConditionNOT extends ConditionItem { + + private int argCount; + private boolean operator; + + public ConditionNOT(boolean tokenList, List> args) { + super(1, tokenList, args); + this.argCount = 1; + this.operator = true; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionOR.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionOR.java new file mode 100644 index 0000000000..b71ab5bfc7 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionOR.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class ConditionOR extends ConditionItem { + + private int argCount; + private boolean operator; + + public ConditionOR(boolean tokenList, List> args) { + super(2, tokenList, args); + this.argCount = 2; + this.operator = true; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionSelector.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionSelector.java new file mode 100644 index 0000000000..6d297ccfee --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionSelector.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetections; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +public class ConditionSelector { + + private int argCount; + private boolean tokenList; + private Either, Class> condClass; + private String pattern; + + private Either parent; + private boolean operator; + + public ConditionSelector(String quantifier, String identifierPattern) { + this.argCount = 2; + this.tokenList = true; + + if ("1".equals(quantifier) || "any".equals(quantifier)) { + this.condClass = Either.right(ConditionOR.class); + } else { + this.condClass = Either.left(ConditionAND.class); + } + this.pattern = identifierPattern; + } + + public ConditionItem postProcess(SigmaDetections detections, Object parent) throws SigmaConditionError { + this.parent = parent instanceof ConditionItem? Either.left((ConditionItem) parent): Either.right((SigmaDetectionItem) parent); + + Pattern r; + if (this.pattern.equals("them")) { + r = Pattern.compile(".*"); + } else { + r = Pattern.compile(this.pattern.replace("*", ".*")); + } + + List> ids = new ArrayList<>(); + for (String identifier: detections.getDetections().keySet()) { + if (r.matcher(identifier).matches()) { + ConditionItem item = new ConditionIdentifier(List.of(Either.right(identifier))).postProcess(detections, parent); + ids.add(Either.left(item)); + } + } + + ConditionItem conditionItem; + if (this.condClass.isLeft()) { + conditionItem = new ConditionAND(false, ids); + } else { + conditionItem = new ConditionOR(false, ids); + } + return conditionItem; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionTraverseVisitor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionTraverseVisitor.java new file mode 100644 index 0000000000..76b6ab341e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionTraverseVisitor.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaCondition; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionBaseVisitor; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionParser; + +import java.util.List; + +public class ConditionTraverseVisitor extends ConditionBaseVisitor> { + + private final SigmaCondition sigmaCondition; + + public ConditionTraverseVisitor(SigmaCondition sigmaCondition) { + this.sigmaCondition = sigmaCondition; + } + + @Override + public Either visitStart(ConditionParser.StartContext ctx) { + return super.visit(ctx.expression()); + } + + @Override + public Either visitIdentOrSelectExpression(ConditionParser.IdentOrSelectExpressionContext ctx) { + if (ctx.IDENTIFIER() != null) { + return Either.right(ctx.IDENTIFIER().getText()); + } + return Either.right(ctx.SELECTOR().getText()); + } + + @Override + public Either visitNotExpression(ConditionParser.NotExpressionContext ctx) { + try { + Either exp = visit(ctx.expression()); + ConditionNOT condition = new ConditionNOT(false, List.of(exp.isLeft()? Either.left(exp.getLeft()): Either.right(exp.get()))); + condition.setArgs(sigmaCondition.convertArgs(condition.getArgs())); + return Either.left(condition); + } catch (SigmaConditionError ex) { + return null; + } + } + + @Override + public Either visitAndExpression(ConditionParser.AndExpressionContext ctx) { + try { + Either left = visit(ctx.left); + Either right = visit(ctx.right); + ConditionAND condition = new ConditionAND(false, List.of( + left.isLeft()? Either.left(left.getLeft()): Either.right(left.get()), + right.isLeft()? Either.left(right.getLeft()): Either.right(right.get()))); + condition.setArgs(sigmaCondition.convertArgs(condition.getArgs())); + return Either.left(condition); + } catch (SigmaConditionError ex) { + return null; + } + } + + @Override + public Either visitOrExpression(ConditionParser.OrExpressionContext ctx) { + try { + Either left = visit(ctx.left); + Either right = visit(ctx.right); + ConditionOR condition = new ConditionOR(false, List.of( + left.isLeft()? Either.left(left.getLeft()): Either.right(left.get()), + right.isLeft()? Either.left(right.getLeft()): Either.right(right.get()))); + condition.setArgs(sigmaCondition.convertArgs(condition.getArgs())); + return Either.left(condition); + } catch (SigmaConditionError ex) { + return null; + } + } + + @Override + public Either visitParenExpression(ConditionParser.ParenExpressionContext ctx) { + return super.visit(ctx.expression()); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionType.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionType.java new file mode 100644 index 0000000000..9843a28d58 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionType.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +public class ConditionType { + + private Either, Either> condition; + + public ConditionType(Either, Either> condition) { + this.condition = condition; + } + + public ConditionAND getConditionAND() { + return this.condition.getLeft().getLeft(); + } + + public boolean isConditionAND() { + return this.condition.isLeft() && this.condition.getLeft().isLeft(); + } + + public ConditionOR getConditionOR() { + return this.condition.getLeft().getMiddle(); + } + + public boolean isConditionOR() { + return this.condition.isLeft() && this.condition.getLeft().isMiddle(); + } + + public ConditionNOT getConditionNOT() { + return this.condition.getLeft().get(); + } + + public boolean isConditionNOT() { + return this.condition.isLeft() && this.condition.getLeft().isRight(); + } + + public ConditionFieldEqualsValueExpression getEqualsValueExpression() { + return this.condition.get().getLeft(); + } + + public boolean isEqualsValueExpression() { + return this.condition.isRight() && this.condition.get().isLeft(); + } + + public ConditionValueExpression getValueExpression() { + return this.condition.get().get(); + } + + public boolean isValueExpression() { + return this.condition.isRight() && this.condition.get().isRight(); + } + + public Class getClazz() { + if (this.condition.isLeft() && this.condition.getLeft().isLeft()) { + return ConditionAND.class; + } else if (this.condition.isLeft() && this.condition.getLeft().isMiddle()) { + return ConditionOR.class; + } else if (this.condition.isLeft() && this.condition.getLeft().isRight()) { + return ConditionNOT.class; + } else if (this.condition.isRight() && this.condition.get().isLeft()) { + return ConditionFieldEqualsValueExpression.class; + } else { + return ConditionValueExpression.class; + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionValueExpression.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionValueExpression.java new file mode 100644 index 0000000000..88b9d32a97 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/condition/ConditionValueExpression.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.condition; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetections; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.Collections; + +public class ConditionValueExpression extends ConditionItem { + + private SigmaType value; + + private Either parent; + + public ConditionValueExpression(SigmaType value) { + super(1, false, Collections.emptyList()); + this.value = value; + } + + public ConditionValueExpression postProcess(SigmaDetections detections, Object parent) { + this.parent = parent instanceof ConditionItem? Either.left((ConditionItem) parent): Either.right((SigmaDetectionItem) parent); + return this; + } + + public SigmaType getValue() { + return value; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaConditionError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaConditionError.java new file mode 100644 index 0000000000..02012dbe29 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaConditionError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaConditionError extends SigmaError { + + public SigmaConditionError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDateError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDateError.java new file mode 100644 index 0000000000..242afbb4fc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDateError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaDateError extends SigmaError { + + public SigmaDateError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDetectionError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDetectionError.java new file mode 100644 index 0000000000..3c7ea7cc14 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaDetectionError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaDetectionError extends SigmaError { + + public SigmaDetectionError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaError.java new file mode 100644 index 0000000000..797daa9dbc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaError.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaError extends RuntimeException { + + private String message; + + public SigmaError(String message) { + super(message); + this.message = message; + } + + @Override + public String toString() { + return this.message; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaIdentifierError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaIdentifierError.java new file mode 100644 index 0000000000..db136e12b1 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaIdentifierError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaIdentifierError extends SigmaError { + + public SigmaIdentifierError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLevelError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLevelError.java new file mode 100644 index 0000000000..93ce2acc60 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLevelError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaLevelError extends SigmaError { + + public SigmaLevelError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLogsourceError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLogsourceError.java new file mode 100644 index 0000000000..dd076e4d47 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaLogsourceError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaLogsourceError extends SigmaError { + + public SigmaLogsourceError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaModifierError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaModifierError.java new file mode 100644 index 0000000000..637e5b8720 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaModifierError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaModifierError extends SigmaError { + + public SigmaModifierError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaRegularExpressionError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaRegularExpressionError.java new file mode 100644 index 0000000000..9acfe8bf45 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaRegularExpressionError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaRegularExpressionError extends SigmaError { + + public SigmaRegularExpressionError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaStatusError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaStatusError.java new file mode 100644 index 0000000000..2d667b1520 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaStatusError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaStatusError extends SigmaError { + + public SigmaStatusError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaTypeError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaTypeError.java new file mode 100644 index 0000000000..0da555f23e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaTypeError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaTypeError extends SigmaModifierError { + + public SigmaTypeError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaValueError.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaValueError.java new file mode 100644 index 0000000000..fabb78bd97 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/exceptions/SigmaValueError.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.exceptions; + +public class SigmaValueError extends SigmaError { + + public SigmaValueError(String message) { + super(message); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaAllModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaAllModifier.java new file mode 100644 index 0000000000..c5e39871d6 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaAllModifier.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionAND; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.List; + +public class SigmaAllModifier extends SigmaListModifier { + + public SigmaAllModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(ArrayList.class, null); + } + + @Override + public Either> modify(Either> val) { + this.getDetectionItem().setValueLinking(Either.left(ConditionAND.class)); + return val; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64Modifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64Modifier.java new file mode 100644 index 0000000000..ab10e98f23 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64Modifier.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.Base64; +import java.util.List; + +public class SigmaBase64Modifier extends SigmaValueModifier { + + public SigmaBase64Modifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, null); + } + + @Override + public Either> modify(Either> val) throws SigmaValueError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + if (((SigmaString) val.getLeft()).containsSpecial()) { + throw new SigmaValueError("Base64 encoding of strings with wildcards is not allowed"); + } + return Either.left(new SigmaString(Base64.getEncoder().encodeToString(((SigmaString) val.getLeft()).getBytes()))); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64OffsetModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64OffsetModifier.java new file mode 100644 index 0000000000..f3270a6bd8 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaBase64OffsetModifier.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaExpansion; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; + +public class SigmaBase64OffsetModifier extends SigmaValueModifier { + + private List startOffsets; + private List endOffsets; + + public SigmaBase64OffsetModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + this.startOffsets = Arrays.asList(0, 2, 3); + this.endOffsets = Arrays.asList(0, -3, -2); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, null); + } + + @Override + public Either> modify(Either> val) throws SigmaValueError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + if (((SigmaString) val.getLeft()).containsSpecial()) { + throw new SigmaValueError("Base64 encoding of strings with wildcards is not allowed"); + } + + List values = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + byte[] valBytes = ((SigmaString) val.getLeft()).getBytes(); + + for (int j = 0; j < i; ++j) { + valBytes = ArrayUtils.insert(0, valBytes, (byte) ' '); + } + String valB64Encode = Base64.getEncoder().encodeToString(valBytes); + valB64Encode = valB64Encode.substring(startOffsets.get(i), valB64Encode.length() + endOffsets.get((((SigmaString) val.getLeft()).length() + i) % 3)); + values.add(new SigmaString(valB64Encode)); + } + + return Either.left(new SigmaExpansion(values)); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCIDRModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCIDRModifier.java new file mode 100644 index 0000000000..448e47d66e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCIDRModifier.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaTypeError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCIDRExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaCIDRModifier extends SigmaValueModifier { + + public SigmaCIDRModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, null); + } + + @Override + public Either> modify(Either> val) throws SigmaValueError, SigmaTypeError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + if (this.getAppliedModifiers().size() > 0) { + throw new SigmaValueError("CIDR expression modifier only applicable to unmodified values"); + } + return Either.left(new SigmaCIDRExpression(val.getLeft().toString())); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCompareModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCompareModifier.java new file mode 100644 index 0000000000..968b9ac48e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaCompareModifier.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCompareExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaFloat; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaCompareModifier extends SigmaValueModifier { + private String op; + + public SigmaCompareModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + public void setOp(String op) { + this.op = op; + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaFloat.class, SigmaCompareExpression.class); + } + + @Override + public Either> modify(Either> val) { + if (val.isLeft()) { + return Either.left(new SigmaCompareExpression((SigmaFloat) val.getLeft(), this.op)); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaContainsModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaContainsModifier.java new file mode 100644 index 0000000000..724e9ca19d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaContainsModifier.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaRegularExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaContainsModifier extends SigmaValueModifier { + + public SigmaContainsModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, SigmaRegularExpression.class); + } + + @Override + public Either> modify(Either> val) throws SigmaRegularExpressionError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + SigmaString value = (SigmaString) val.getLeft(); + if (!value.startsWith(Either.right(SigmaString.SpecialChars.WILDCARD_MULTI))) { + value.prepend(AnyOneOf.middleVal(SigmaString.SpecialChars.WILDCARD_MULTI)); + } + if (!value.endsWith(Either.right(SigmaString.SpecialChars.WILDCARD_MULTI))) { + value.append(AnyOneOf.middleVal(SigmaString.SpecialChars.WILDCARD_MULTI)); + } + val = Either.left(value); + return val; + } else if (val.isLeft() && val.getLeft() instanceof SigmaRegularExpression) { + SigmaRegularExpression value = (SigmaRegularExpression) val.getLeft(); + if (!value.getRegexp().startsWith(".*") && value.getRegexp().charAt(0) != '^') { + value.setRegexp(".*" + value.getRegexp()); + } + if (!value.getRegexp().endsWith(".*") && !value.getRegexp().endsWith("$")) { + value.setRegexp(value.getRegexp() + ".*"); + } + value.compile(); + return Either.left(value); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaEndswithModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaEndswithModifier.java new file mode 100644 index 0000000000..060617a471 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaEndswithModifier.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaRegularExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaEndswithModifier extends SigmaValueModifier { + + public SigmaEndswithModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, SigmaRegularExpression.class); + } + + @Override + public Either> modify(Either> val) throws SigmaRegularExpressionError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + SigmaString value = (SigmaString) val.getLeft(); + if (!value.startsWith(Either.right(SigmaString.SpecialChars.WILDCARD_MULTI))) { + value.prepend(AnyOneOf.middleVal(SigmaString.SpecialChars.WILDCARD_MULTI)); + } + val = AnyOneOf.leftVal(value); + return val; + } else if (val.isLeft() && val.getLeft() instanceof SigmaRegularExpression) { + SigmaRegularExpression value = (SigmaRegularExpression) val.getLeft(); + if (!value.getRegexp().startsWith(".*") && value.getRegexp().charAt(0) != '^') { + value.setRegexp(".*" + value.getRegexp()); + } + value.compile(); + return Either.left(value); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanEqualModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanEqualModifier.java new file mode 100644 index 0000000000..2e81ab09dd --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanEqualModifier.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCompareExpression; + +import java.util.List; + +public class SigmaGreaterThanEqualModifier extends SigmaCompareModifier { + + public SigmaGreaterThanEqualModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + this.setOp(SigmaCompareExpression.CompareOperators.GTE); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanModifier.java new file mode 100644 index 0000000000..a09efd7131 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaGreaterThanModifier.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCompareExpression; + +import java.util.List; + +public class SigmaGreaterThanModifier extends SigmaCompareModifier { + + public SigmaGreaterThanModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + this.setOp(SigmaCompareExpression.CompareOperators.GT); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanEqualModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanEqualModifier.java new file mode 100644 index 0000000000..d5db10aab0 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanEqualModifier.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCompareExpression; + +import java.util.List; + +public class SigmaLessThanEqualModifier extends SigmaCompareModifier { + + public SigmaLessThanEqualModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + this.setOp(SigmaCompareExpression.CompareOperators.LTE); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanModifier.java new file mode 100644 index 0000000000..51b43892cc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaLessThanModifier.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaCompareExpression; + +import java.util.List; + +public class SigmaLessThanModifier extends SigmaCompareModifier { + + public SigmaLessThanModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + this.setOp(SigmaCompareExpression.CompareOperators.LT); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaListModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaListModifier.java new file mode 100644 index 0000000000..77454af7e5 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaListModifier.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public abstract class SigmaListModifier extends SigmaModifier { + + + public SigmaListModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + public abstract Either> modify(Either> val); +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifier.java new file mode 100644 index 0000000000..c9520a6c32 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifier.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaTypeError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaExpansion; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class SigmaModifier { + + private SigmaDetectionItem detectionItem; + + private List> appliedModifiers; + + public SigmaModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + this.detectionItem = detectionItem; + this.appliedModifiers = appliedModifiers; + } + + public boolean typeCheck(Either> val) { + Pair, Class> typePair = this.getTypeHints(); + return (typePair.getLeft() != null && val.isLeft() && typePair.getLeft().equals(val.getLeft().getClass())) || + (typePair.getRight() != null && val.isLeft() && typePair.getRight().equals(val.getLeft().getClass())) || + (typePair.getLeft() != null && val.isRight() && typePair.getLeft().equals(val.get().getClass())) || + (typePair.getRight() != null && val.isRight() && typePair.getRight().equals(val.get().getClass())); + } + + public abstract Either> modify(Either> val) throws SigmaValueError, SigmaRegularExpressionError, SigmaTypeError; + + public abstract Pair, Class> getTypeHints(); + + public List apply(Either> val) throws SigmaTypeError, SigmaValueError, SigmaRegularExpressionError { + if (val.isLeft() && val.getLeft() instanceof SigmaExpansion) { + List values = new ArrayList<>(); + for (SigmaType value: ((SigmaExpansion) val.getLeft()).getValues()) { + List va = this.apply(Either.left(value)); + values.addAll(va); + } + return Collections.singletonList(new SigmaExpansion(values)); + } else { + if (!this.typeCheck(val)) { + throw new SigmaTypeError("Modifier " + this.getClass().getName() + " incompatible to value type of '" + val + "'"); + } + Either> r = this.modify(val); + if (r.isRight()) { + return r.get(); + } else { + return Collections.singletonList(r.getLeft()); + } + } + } + + public SigmaDetectionItem getDetectionItem() { + return detectionItem; + } + + public List> getAppliedModifiers() { + return appliedModifiers; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifierFacade.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifierFacade.java new file mode 100644 index 0000000000..6e7a57c40b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaModifierFacade.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaModifierError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SigmaModifierFacade { + + private Map> modifierMap; + private static SigmaModifierFacade modifierFacade; + + public SigmaModifierFacade() { + modifierMap = new HashMap<>(); + modifierMap.put("contains", SigmaContainsModifier.class); + modifierMap.put("startswith", SigmaStartswithModifier.class); + modifierMap.put("endswith", SigmaEndswithModifier.class); + modifierMap.put("base64", SigmaBase64Modifier.class); + modifierMap.put("base64offset", SigmaBase64OffsetModifier.class); + modifierMap.put("wide", SigmaWideModifier.class); + modifierMap.put("windash", SigmaWindowsDashModifier.class); + modifierMap.put("re", SigmaRegularExpressionModifier.class); + modifierMap.put("cidr", SigmaCIDRModifier.class); + modifierMap.put("all", SigmaAllModifier.class); + modifierMap.put("lt", SigmaLessThanModifier.class); + modifierMap.put("lte", SigmaLessThanEqualModifier.class); + modifierMap.put("gt", SigmaGreaterThanModifier.class); + modifierMap.put("gte", SigmaGreaterThanEqualModifier.class); + } + + public static SigmaModifier sigmaModifier(Class clazz, SigmaDetectionItem detectionItem, + List> appliedModifiers) throws SigmaModifierError { + if (modifierFacade == null) { + modifierFacade = new SigmaModifierFacade(); + } + + if (clazz.equals(SigmaContainsModifier.class)) { + return new SigmaContainsModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaStartswithModifier.class)) { + return new SigmaStartswithModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaEndswithModifier.class)) { + return new SigmaEndswithModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaBase64Modifier.class)) { + return new SigmaBase64Modifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaBase64OffsetModifier.class)) { + return new SigmaBase64OffsetModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaWideModifier.class)) { + return new SigmaWideModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaWindowsDashModifier.class)) { + return new SigmaWindowsDashModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaRegularExpressionModifier.class)) { + return new SigmaRegularExpressionModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaCIDRModifier.class)) { + return new SigmaCIDRModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaAllModifier.class)) { + return new SigmaAllModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaLessThanModifier.class)) { + return new SigmaLessThanModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaLessThanEqualModifier.class)) { + return new SigmaLessThanEqualModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaGreaterThanModifier.class)) { + return new SigmaGreaterThanModifier(detectionItem, appliedModifiers); + } else if (clazz.equals(SigmaGreaterThanEqualModifier.class)) { + return new SigmaGreaterThanEqualModifier(detectionItem, appliedModifiers); + } + throw new SigmaModifierError("modifier not found-" + clazz.getName()); + } + + public static Map reverseModifierMapping() { + if (modifierFacade == null) { + modifierFacade = new SigmaModifierFacade(); + } + + Map reverseModifierMap = new HashMap<>(); + for (Map.Entry> modifier: modifierFacade.modifierMap.entrySet()) { + reverseModifierMap.put(modifier.getValue().getName(), modifier.getKey()); + } + + return reverseModifierMap; + } + + public static Class getModifier(String modifier) { + if (modifierFacade == null) { + modifierFacade = new SigmaModifierFacade(); + } + return modifierFacade.modifierMap.get(modifier); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaRegularExpressionModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaRegularExpressionModifier.java new file mode 100644 index 0000000000..6191fe01b1 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaRegularExpressionModifier.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaRegularExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaRegularExpressionModifier extends SigmaValueModifier { + + public SigmaRegularExpressionModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaRegularExpression.class, SigmaString.class); + } + + @Override + public Either> modify(Either> val) throws SigmaValueError, SigmaRegularExpressionError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + if (this.getAppliedModifiers().size() > 0) { + throw new SigmaValueError("Regular expression modifier only applicable to unmodified values"); + } + return Either.left(new SigmaRegularExpression(((SigmaString) val.getLeft()).getOriginal())); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaStartswithModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaStartswithModifier.java new file mode 100644 index 0000000000..1497898bd0 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaStartswithModifier.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaRegularExpression; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public class SigmaStartswithModifier extends SigmaValueModifier { + + public SigmaStartswithModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, SigmaRegularExpression.class); + } + + @Override + public Either> modify(Either> val) throws SigmaRegularExpressionError { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + SigmaString value = (SigmaString) val.getLeft(); + if (!value.endsWith(Either.right(SigmaString.SpecialChars.WILDCARD_MULTI))) { + value.append(AnyOneOf.middleVal(SigmaString.SpecialChars.WILDCARD_MULTI)); + } + val = Either.left(value); + return val; + } else if (val.isLeft() && val.getLeft() instanceof SigmaRegularExpression) { + SigmaRegularExpression value = (SigmaRegularExpression) val.getLeft(); + if (!value.getRegexp().endsWith(".*") && !value.getRegexp().endsWith("$")) { + value.setRegexp(value.getRegexp() + ".*"); + } + value.compile(); + return Either.left(value); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaValueModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaValueModifier.java new file mode 100644 index 0000000000..73ca245ae0 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaValueModifier.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaTypeError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; + +public abstract class SigmaValueModifier extends SigmaModifier { + + public SigmaValueModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + public abstract Either> modify(Either> val) throws SigmaValueError, SigmaRegularExpressionError, SigmaTypeError; +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWideModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWideModifier.java new file mode 100644 index 0000000000..b29d9d49fa --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWideModifier.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.Placeholder; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public class SigmaWideModifier extends SigmaValueModifier { + + public SigmaWideModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, null); + } + + @Override + public Either> modify(Either> val) { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + List> r = new ArrayList<>(); + + for (AnyOneOf sOptElem: ((SigmaString) val.getLeft()).getsOpt()) { + if (sOptElem.isLeft()) { + r.add(AnyOneOf.leftVal(new String(sOptElem.getLeft().getBytes(StandardCharsets.UTF_16LE), StandardCharsets.UTF_8))); + } else{ + r.add(sOptElem); + } + } + + SigmaString s = new SigmaString(""); + s.setsOpt(r); + return Either.left(s); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWindowsDashModifier.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWindowsDashModifier.java new file mode 100644 index 0000000000..7f74d3df82 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/modifiers/SigmaWindowsDashModifier.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.modifiers; + +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaDetectionItem; +import org.opensearch.dataprepper.plugins.processor.parser.types.Placeholder; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaExpansion; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaString; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.List; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class SigmaWindowsDashModifier extends SigmaValueModifier { + + public SigmaWindowsDashModifier(SigmaDetectionItem detectionItem, List> appliedModifiers) { + super(detectionItem, appliedModifiers); + } + + @Override + public Pair, Class> getTypeHints() { + return Pair.of(SigmaString.class, SigmaExpansion.class); + } + + @Override + public Either> modify(Either> val) { + if (val.isLeft() && val.getLeft() instanceof SigmaString) { + Function>> callback = + p -> { + if (p.getName().equals("_windash")) { + return List.of(AnyOneOf.leftVal("-"), AnyOneOf.leftVal("/")); + } + return List.of(AnyOneOf.rightVal(p)); + }; + return Either.left(new SigmaExpansion(new SigmaString(val.getLeft().toString().replace("_ws_", " ")).replaceWithPlaceholder(Pattern.compile("\\B[-/]\\b"), "_windash") + .replacePlaceholders(callback).stream().map(s -> new SigmaString(s.toString().replace(" ", "_ws_"))).collect(Collectors.toList()))); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java new file mode 100644 index 0000000000..55e5305e76 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionFieldEqualsValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionIdentifier; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionSelector; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionTraverseVisitor; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionLexer; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionParser; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class SigmaCondition { + + private final String identifier = "[a-zA-Z0-9-_]+"; + + private final List quantifier = List.of("1", "any", "all"); + + private final String identifierPattern = "[a-zA-Z0-9*_]+"; + + private final List, String>> selector = List.of(Either.left(quantifier), Either.right("of"), Either.right(identifierPattern)); + + private final List operators = List.of("not ", " and ", " or "); + + private String condition; + + private String aggregation; + + private SigmaDetections detections; + + private ConditionParser parser; + + private ConditionTraverseVisitor conditionVisitor; + + public SigmaCondition(String condition, SigmaDetections detections) { + if (condition.contains(" | ")) { + this.condition = condition.split(" \\| ")[0]; + this.aggregation = condition.split(" \\| ")[1]; + } else { + this.condition = condition; + this.aggregation = ""; + } + + this.detections = detections; + + ConditionLexer lexer = new ConditionLexer(CharStreams.fromString(this.condition)); + this.parser = new ConditionParser(new CommonTokenStream(lexer)); + this.conditionVisitor = new ConditionTraverseVisitor(this); + } + + public ConditionItem parsed() throws SigmaConditionError { + ConditionItem parsedConditionItem; + Either itemOrCondition = conditionVisitor.visit(parser.start()); + if (itemOrCondition.isLeft()) { + parsedConditionItem = itemOrCondition.getLeft(); + } else { + parsedConditionItem = Objects.requireNonNull(parsed(condition)); + } + + return parsedConditionItem; + } + + public List> convertArgs(List> parsedArgs) throws SigmaConditionError { + List> newArgs = new ArrayList<>(); + + for (Either parsedArg: parsedArgs) { + if (parsedArg.isRight()) { + ConditionItem newItem = parsed(parsedArg.get()); + newArgs.add(Either.left(newItem)); + } else { + newArgs.add(parsedArg); + } + } + return newArgs; + } + + private ConditionItem parsed(String token) throws SigmaConditionError { + List subTokens = List.of(token.split(" ")); + if (subTokens.size() < 3 && token.matches(identifier)) { + ConditionIdentifier conditionIdentifier = + new ConditionIdentifier(Collections.singletonList(Either.right(token))); + ConditionItem item = conditionIdentifier.postProcess(detections, null); + return item instanceof ConditionFieldEqualsValueExpression? (ConditionFieldEqualsValueExpression) item: + (item instanceof ConditionValueExpression ? (ConditionValueExpression) item: item); + } else if (subTokens.size() == 3 && quantifier.contains(subTokens.get(0)) && selector.get(1).get().equals(subTokens.get(1)) && + subTokens.get(2).matches(identifierPattern)) { + ConditionSelector conditionSelector = + new ConditionSelector(subTokens.get(0), subTokens.get(2)); + ConditionItem item = conditionSelector.postProcess(detections, null); + return item instanceof ConditionFieldEqualsValueExpression? (ConditionFieldEqualsValueExpression) item: + (item instanceof ConditionValueExpression ? (ConditionValueExpression) item: item); + } + return null; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetection.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetection.java new file mode 100644 index 0000000000..024933219e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetection.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionAND; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionFieldEqualsValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionOR; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaDetectionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaModifierError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SigmaDetection { + + private List> detectionItems; + private Either, Class> itemLinking; + + private Either parent; + + public SigmaDetection(List> detectionItems, + Either, Class> itemLinking) throws SigmaDetectionError { + this.detectionItems = detectionItems; + this.itemLinking = itemLinking == null? Either.left(ConditionAND.class): itemLinking; + + if (this.detectionItems.size() == 0) { + throw new SigmaDetectionError("Detection is empty"); + } + + List> typeSet = new ArrayList<>(); + for (Either detectionItem: detectionItems) { + if (detectionItem.isLeft()) { + typeSet.add(SigmaDetectionItem.class); + } + if (detectionItem.isRight()) { + typeSet.add(SigmaDetection.class); + } + } + + if (typeSet.contains(SigmaDetectionItem.class)) { + this.itemLinking = Either.left(ConditionAND.class); + } else { + this.itemLinking = Either.right(ConditionOR.class); + } + } + + @SuppressWarnings("unchecked") + protected static SigmaDetection fromDefinition(Object definition) throws SigmaModifierError, SigmaDetectionError, SigmaValueError, SigmaRegularExpressionError { + List> detectionItems = new ArrayList<>(); + if (definition instanceof Map) { + for (Map.Entry defEntry: ((Map) definition).entrySet()) { + Object val = defEntry.getValue(); + + if (val == null) { + detectionItems.add(Either.left(SigmaDetectionItem.fromMapping(defEntry.getKey(), + Either.left(null)))); + } else if (val instanceof Integer) { + detectionItems.add(Either.left(SigmaDetectionItem.fromMapping(defEntry.getKey(), + Either.left((Integer) val)))); + } else if (val instanceof Float) { + detectionItems.add(Either.left(SigmaDetectionItem.fromMapping(defEntry.getKey(), + Either.left((Float) val)))); + } else if (val instanceof String) { + detectionItems.add(Either.left(SigmaDetectionItem.fromMapping(defEntry.getKey(), + Either.left(val.toString())))); + } else if (val instanceof Boolean) { + detectionItems.add(Either.left(SigmaDetectionItem.fromMapping(defEntry.getKey(), + Either.left((Boolean) val)))); + } else if (val instanceof List) { + SigmaDetectionItem item = + SigmaDetectionItem.fromMapping(defEntry.getKey(), Either.right(((List) val))); + detectionItems.add(Either.left(item)); + } + } + return new SigmaDetection(detectionItems, null); + } else if (definition instanceof String || definition instanceof Integer) { + detectionItems.add(Either.left(SigmaDetectionItem.fromValue(Either.left(definition)))); + return new SigmaDetection(detectionItems, null); + } else if (definition instanceof ArrayList) { + List definitionList = (List) definition; + + boolean isItem = true; + for (Object definitionElem: definitionList) { + if (!(definitionElem instanceof String) && !(definitionElem instanceof Integer)) { + detectionItems.add(Either.right(SigmaDetection.fromDefinition(definitionElem))); + isItem = false; + } + } + + if (isItem) { + detectionItems.add(Either.left(SigmaDetectionItem.fromValue(Either.right(definitionList)))); + return new SigmaDetection(detectionItems, null); + } + return new SigmaDetection(detectionItems, null); + } + throw new SigmaValueError("Unexpected Values"); + } + + public ConditionItem postProcess(SigmaDetections detections, Object parent) throws SigmaConditionError { + this.parent = parent instanceof ConditionItem? Either.left((ConditionItem) parent): Either.right((SigmaDetection)parent); + + List> valueExpressions = new ArrayList<>(); + for (Either detectionItem: this.detectionItems) { + if (detectionItem.isLeft()) { + Either, Either> item = + detectionItem.getLeft().postProcess(detections, this); + + if (item.isLeft() && item.getLeft().isLeft()) { + valueExpressions.add(Either.left(item.getLeft().getLeft())); + } else if (item.isLeft() && item.getLeft().isRight()) { + valueExpressions.add(Either.left(item.getLeft().get())); + } else if (item.isRight() && item.get().isLeft()) { + valueExpressions.add(Either.left(item.get().getLeft())); + } else if (item.isRight() && item.get().isRight()) { + valueExpressions.add(Either.left(item.get().get())); + } + } else if (detectionItem.isRight()) { + ConditionItem item = detectionItem.get().postProcess(detections, this); + valueExpressions.add(Either.left(item)); + } + } + + if (valueExpressions.size() == 1) { + return valueExpressions.get(0).getLeft(); + } else { + if (itemLinking.isLeft()) { + return new ConditionAND(false, valueExpressions); + } else if (itemLinking.isRight()) { + return new ConditionOR(false, valueExpressions); + } + } + return null; + } + + public List> getDetectionItems() { + return detectionItems; + } + + public Either, Class> getItemLinking() { + return itemLinking; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetectionItem.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetectionItem.java new file mode 100644 index 0000000000..e8ff51d0c9 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetectionItem.java @@ -0,0 +1,198 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionAND; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionFieldEqualsValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionOR; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaModifierError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.modifiers.SigmaListModifier; +import org.opensearch.dataprepper.plugins.processor.parser.modifiers.SigmaModifier; +import org.opensearch.dataprepper.plugins.processor.parser.modifiers.SigmaModifierFacade; +import org.opensearch.dataprepper.plugins.processor.parser.modifiers.SigmaValueModifier; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaNull; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaType; +import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaTypeFacade; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class SigmaDetectionItem { + + private String field; + + private List> modifiers; + + private List value; + + private Either, Class> valueLinking; + + private SigmaType originalValue; + + private boolean autoModifiers; + + private SigmaDetection parent; + + public SigmaDetectionItem(String field, List> modifiers, List value, + Either, Class> valueLinking, SigmaType originalValue, boolean autoModifiers) throws SigmaModifierError, SigmaValueError, SigmaRegularExpressionError { + this.field = field; + this.modifiers = modifiers; + this.value = value; + this.valueLinking = valueLinking != null? valueLinking: Either.right(ConditionOR.class); + this.originalValue = originalValue; + this.autoModifiers = autoModifiers; + + if (autoModifiers) { + this.applyModifiers(); + } + } + + private void applyModifiers() throws SigmaModifierError, SigmaValueError, SigmaRegularExpressionError { + List> appliedModifiers = new ArrayList<>(); + + for (Class modifier: modifiers) { + SigmaModifier modifierInstance = SigmaModifierFacade.sigmaModifier(modifier, this, appliedModifiers); + + if (modifierInstance instanceof SigmaValueModifier) { + List appliedValue = new ArrayList<>(); + for (SigmaType val: this.value) { + appliedValue.addAll(modifierInstance.apply(Either.left(val))); + } + this.value = appliedValue; + } else if (modifierInstance instanceof SigmaListModifier) { + this.value = modifierInstance.apply(Either.right(this.value)); + } else { + throw new IllegalArgumentException("Instance of SigmaValueModifier or SigmaListModifier was expected"); + } + appliedModifiers.add(modifier); + } + } + + public static SigmaDetectionItem fromMapping(String key, Either> val) throws SigmaModifierError, SigmaValueError, SigmaRegularExpressionError { + String field = null; + List modifierIds = new ArrayList<>(); + if (key != null) { + String[] tokens = key.split("\\|"); + if (tokens.length > 0) { + field = tokens[0].isEmpty()? null: tokens[0]; + modifierIds = Arrays.stream(tokens).skip(1).collect(Collectors.toList()); + } + } + + List> modifiers = new ArrayList<>(); + for (String modId: modifierIds) { + Class modifier = SigmaModifierFacade.getModifier(modId); + if (modifier != null) { + modifiers.add(modifier); + } else { + throw new SigmaModifierError("Unknown modifier " + modId); + } + } + + List values = new ArrayList<>(); + if (val != null && val.isLeft()) { + values.add(val.getLeft()); + } else if (val != null && val.isRight()) { + values.addAll(val.get()); + } else { + values.add(null); + } + + List sigmaTypes = new ArrayList<>(); + for (T v: values) { + sigmaTypes.add(SigmaTypeFacade.sigmaType(v)); + } + + return new SigmaDetectionItem(field, modifiers, sigmaTypes, null, null, true); + } + + public static SigmaDetectionItem fromValue(Either> val) throws SigmaModifierError, SigmaValueError, SigmaRegularExpressionError { + return SigmaDetectionItem.fromMapping(null, val); + } + + public Either, Either> postProcess(SigmaDetections detections, Object parent) throws SigmaConditionError { + this.parent = (SigmaDetection) parent; + + if (this.value.size() == 0) { + if (this.field == null) { + throw new SigmaConditionError("Null value must be bound to a field"); + } else { + return Either.right(Either.left(new ConditionFieldEqualsValueExpression(field, new SigmaNull()).postProcess(detections, this))); + } + } + if (this.value.size() == 1) { + if (this.field == null) { + return Either.right(Either.right(new ConditionValueExpression(this.value.get(0)).postProcess(detections, this))); + } else { + return Either.right(Either.left(new ConditionFieldEqualsValueExpression(field, value.get(0)).postProcess(detections, this))); + } + } else { + List> valueExpressions = new ArrayList<>(); + if (this.field == null) { + for (SigmaType v: value) { + valueExpressions.add(Either.left(new ConditionValueExpression(v))); + } + + if (valueLinking.isLeft()) { + ConditionAND conditionAND = new ConditionAND(false, valueExpressions); + conditionAND = (ConditionAND) conditionAND.postProcess(detections, this); + return Either.left(Either.left(conditionAND)); + } else if (valueLinking.isRight()) { + ConditionOR conditionOR = new ConditionOR(false, valueExpressions); + conditionOR = (ConditionOR) conditionOR.postProcess(detections, this); + return Either.left(Either.right(conditionOR)); + } + } else { + for (SigmaType v: value) { + valueExpressions.add(Either.left(new ConditionFieldEqualsValueExpression(field, v))); + } + + if (valueLinking.isLeft()) { + ConditionAND conditionAND = new ConditionAND(false, valueExpressions); + conditionAND = (ConditionAND) conditionAND.postProcess(detections, this); + return Either.left(Either.left(conditionAND)); + } else if (valueLinking.isRight()) { + ConditionOR conditionOR = new ConditionOR(false, valueExpressions); + conditionOR = (ConditionOR) conditionOR.postProcess(detections, this); + return Either.left(Either.right(conditionOR)); + } + } + } + return null; + } + + public boolean isKeyword() { + return field == null; + } + + public String getField() { + return field; + } + + public List getValue() { + return value; + } + + public List> getModifiers() { + return modifiers; + } + + public Either, Class> getValueLinking() { + return valueLinking; + } + + public void setValueLinking(Either, Class> valueLinking) { + this.valueLinking = valueLinking; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetections.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetections.java new file mode 100644 index 0000000000..aabc42bc02 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaDetections.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaDetectionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaModifierError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SigmaDetections { + + private Map detections; + + private List conditions; + + private String timeframe; + + private List parsedConditions; + + public SigmaDetections(Map detections, List conditions, String timeframe) throws SigmaDetectionError { + this.detections = detections; + this.conditions = conditions; + this.timeframe = timeframe; + + if (this.detections.isEmpty()) { + throw new SigmaDetectionError("No detections defined in Sigma rule"); + } + + this.parsedConditions = new ArrayList<>(); + for (String cond: this.conditions) { + this.parsedConditions.add(new SigmaCondition(cond, this)); + } + } + + @SuppressWarnings("unchecked") + protected static SigmaDetections fromDict(Map detectionMap) throws SigmaConditionError, SigmaDetectionError, SigmaModifierError, SigmaValueError, SigmaRegularExpressionError { + List conditionList = new ArrayList<>(); + if (detectionMap.containsKey("condition") && detectionMap.get("condition") instanceof List) { + conditionList.addAll((List) detectionMap.get("condition")); + } else if (detectionMap.containsKey("condition")) { + conditionList.add(detectionMap.get("condition").toString()); + } else { + throw new SigmaConditionError("Sigma rule must contain at least one condition"); + } + + Map detections = new HashMap<>(); + for (Map.Entry detection: detectionMap.entrySet()) { + if (!"condition".equals(detection.getKey())) { + detections.put(detection.getKey(), SigmaDetection.fromDefinition(detection.getValue())); + } + } + + String timeframe = null; + if (detectionMap.containsKey("timeframe")) { + timeframe = detectionMap.get("timeframe").toString(); + } + + return new SigmaDetections(detections, conditionList, timeframe); + } + + public Map getDetections() { + return detections; + } + + public List getConditions() { + return conditions; + } + + public List getParsedConditions() { + return parsedConditions; + } + + public String getTimeframe() { + return timeframe; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLevel.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLevel.java new file mode 100644 index 0000000000..3a18406628 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLevel.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import java.util.Locale; + +public enum SigmaLevel { + INFORMATIONAL, + LOW, + MEDIUM, + HIGH, + CRITICAL; + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLogSource.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLogSource.java new file mode 100644 index 0000000000..83404fc877 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaLogSource.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaLogsourceError; + +import java.util.Map; + +public class SigmaLogSource { + + private String product; + + private String category; + + private String service; + + public SigmaLogSource(String product, String category, String service) throws SigmaLogsourceError { + this.product = product; + this.category = category; + this.service = service; + + if ((this.product == null || this.product.isEmpty()) && (this.category == null || this.category.isEmpty()) + && (this.service == null || this.service.isEmpty())) { + throw new SigmaLogsourceError("Log source can't be empty"); + } + } + + protected static SigmaLogSource fromDict(Map logSource) throws SigmaLogsourceError { + String product = ""; + if (logSource.containsKey("product")) { + product = logSource.get("product").toString(); + } + + String category = ""; + if (logSource.containsKey("category")) { + category = logSource.get("category").toString(); + } + + String service = ""; + if (logSource.containsKey("service")) { + service = logSource.get("service").toString(); + } + return new SigmaLogSource(product, category, service); + } + + public String getProduct() { + return product; + } + + public String getCategory() { + return category; + } + + public String getService() { + return service; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRule.java new file mode 100644 index 0000000000..336c0bc2f7 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRule.java @@ -0,0 +1,237 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaDateError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaDetectionError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaIdentifierError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaLevelError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaLogsourceError; +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaStatusError; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.representer.Representer; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +public class SigmaRule { + + private String title; + + private SigmaLogSource logSource; + + private SigmaDetections detection; + + private UUID id; + + private SigmaStatus status; + + private String description; + + private List references; + + private List tags; + + private String author; + + private Date date; + + private List fields; + + private List falsePositives; + + private SigmaLevel level; + + private List errors; + + public SigmaRule(String title, SigmaLogSource logSource, SigmaDetections detection, UUID id, SigmaStatus status, + String description, List references, List tags, String author, Date date, + List fields, List falsePositives, SigmaLevel level, List errors) { + this.title = title; + this.logSource = logSource; + this.detection = detection; + this.id = id; + this.status = status; + this.description = description; + this.references = references; + this.tags = tags; + this.author = author; + this.date = date; + this.fields = fields; + this.falsePositives = falsePositives; + this.level = level; + + this.errors = errors; + + if (this.references == null) { + this.references = new ArrayList<>(); + } + if (this.tags == null) { + this.tags = new ArrayList<>(); + } + if (this.fields == null) { + this.fields = new ArrayList<>(); + } + if (this.falsePositives == null) { + this.falsePositives = new ArrayList<>(); + } + } + + @SuppressWarnings("unchecked") + protected static SigmaRule fromDict(Map rule, boolean collectErrors) throws SigmaError { + List errors = new ArrayList<>(); + + UUID ruleId; + if (rule.containsKey("id")) { + try { + ruleId = UUID.fromString(rule.get("id").toString()); + } catch (IllegalArgumentException ex) { + errors.add(new SigmaIdentifierError("Sigma rule identifier must be an UUID")); + ruleId = null; + } + } else { + errors.add(new SigmaIdentifierError("Sigma rule identifier must be an UUID")); + ruleId = null; + } + + SigmaLevel level; + if (rule.containsKey("level")) { + level = SigmaLevel.valueOf(rule.get("level").toString().toUpperCase(Locale.ROOT)); + } else { + errors.add(new SigmaLevelError("null is no valid Sigma rule level")); + level = null; + } + + SigmaStatus status; + if (rule.containsKey("status")) { + status = SigmaStatus.valueOf(rule.get("status").toString().toUpperCase(Locale.ROOT)); + } else { + errors.add(new SigmaStatusError("null is no valid Sigma rule status")); + status = null; + } + + Date ruleDate = null; + if (rule.containsKey("date")) { + try { + if (rule.get("date").toString().contains("/")) { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy/MM/dd", Locale.getDefault()); + ruleDate = formatter.parse(rule.get("date").toString()); + } else if (rule.get("date").toString().contains("-")) { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd", Locale.getDefault()); + ruleDate = formatter.parse(rule.get("date").toString()); + } + } catch (Exception ex) { + errors.add(new SigmaDateError("Rule date " + rule.get("date").toString() + " is invalid, must be yyyy/mm/dd or yyyy-mm-dd")); + } + } + + SigmaLogSource logSource; + if (rule.containsKey("logsource")) { + logSource = SigmaLogSource.fromDict((Map) rule.get("logsource")); + } else { + errors.add(new SigmaLogsourceError("Sigma rule must have a log source")); + logSource = null; + } + + SigmaDetections detections; + if (rule.containsKey("detection")) { + detections = SigmaDetections.fromDict((Map) rule.get("detection")); + } else { + errors.add(new SigmaDetectionError("Sigma rule must have a detection definitions")); + detections = null; + } + + List ruleTagsStr = (List) rule.get("tags"); + List ruleTags = new ArrayList<>(); + if (ruleTagsStr != null) { + for (String ruleTag : ruleTagsStr) { + ruleTags.add(SigmaRuleTag.fromStr(ruleTag)); + } + } + + if (!collectErrors && !errors.isEmpty()) { + throw errors.get(0); + } + + return new SigmaRule(rule.get("title").toString(), logSource, detections, ruleId, status, + rule.get("description").toString(), rule.get("references") != null? (List) rule.get("references"): null, ruleTags, + rule.get("author").toString(), ruleDate, rule.get("fields") != null? (List) rule.get("fields"): null, + rule.get("falsepositives") != null? (List) rule.get("falsepositives"): null, level, errors); + } + + public static SigmaRule fromYaml(String rule, boolean collectErrors) throws SigmaError { + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setNestingDepthLimit(10); + + Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions()), new Representer(new DumperOptions()), new DumperOptions(), loaderOptions); + Map ruleMap = yaml.load(rule); + return fromDict(ruleMap, collectErrors); + } + + public String getTitle() { + return title; + } + + public SigmaLogSource getLogSource() { + return logSource; + } + + public SigmaDetections getDetection() { + return detection; + } + + public UUID getId() { + return id; + } + + public SigmaStatus getStatus() { + return status; + } + + public String getDescription() { + return description; + } + + public List getReferences() { + return references; + } + + public List getTags() { + return tags; + } + + public String getAuthor() { + return author; + } + + public Date getDate() { + return date; + } + + public List getFields() { + return fields; + } + + public List getFalsePositives() { + return falsePositives; + } + + public SigmaLevel getLevel() { + return level; + } + + public List getErrors() { + return errors; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRuleTag.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRuleTag.java new file mode 100644 index 0000000000..7acaf1e034 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaRuleTag.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +public class SigmaRuleTag { + + private String namespace; + + private String name; + + public SigmaRuleTag(String namespace, String name) { + this.namespace = namespace; + this.name = name; + } + + public static SigmaRuleTag fromStr(String tag) { + String[] tagParts = tag.split("\\.", 2); + return new SigmaRuleTag(tagParts[0], tagParts[1]); + } + + public String getNamespace() { + return namespace; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return String.format("%s.%s", namespace, name); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaStatus.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaStatus.java new file mode 100644 index 0000000000..d967f5099a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaStatus.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.objects; + +import java.util.Locale; + +public enum SigmaStatus { + STABLE, + EXPERIMENTAL, + TEST, + DEPRECATED, + UNSUPPORTED; + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/Placeholder.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/Placeholder.java new file mode 100644 index 0000000000..256f84494e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/Placeholder.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class Placeholder { + private String name; + + public Placeholder(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaBool.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaBool.java new file mode 100644 index 0000000000..7e66abca45 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaBool.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class SigmaBool implements SigmaType { + + private Boolean aBoolean; + + public SigmaBool(Boolean aBoolean) { + this.aBoolean = aBoolean; + } + + public Boolean getBoolean() { + return aBoolean; + } + + @Override + public String toString() { + return String.valueOf(aBoolean); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCIDRExpression.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCIDRExpression.java new file mode 100644 index 0000000000..2d965436d3 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCIDRExpression.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaTypeError; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SigmaCIDRExpression implements SigmaType { + private String cidr; + + public SigmaCIDRExpression(String cidr) throws SigmaTypeError { + this.cidr = cidr; + + if (!isIPv4AddressValid(this.cidr)) { + throw new SigmaTypeError("Invalid IPv4 CIDR expression"); + } + } + + public String convert() { + return this.cidr; + } + + private static boolean isIPv4AddressValid(String cidr) { + if (cidr == null) { + return false; + } + + String[] values = cidr.split("/"); + Pattern ipv4Pattern = Pattern + .compile("(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.){3}([01]?\\d\\d?|2[0-4]\\d|25[0-5])"); + Matcher mm = ipv4Pattern.matcher(values[0]); + if (!mm.matches()) { + return false; + } + if (values.length >= 2) { + int prefix = Integer.parseInt(values[1]); + if ((prefix < 0) || (prefix > 32)) { + return false; + } + } + return true; + } + + public String getCidr() { + return cidr; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCompareExpression.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCompareExpression.java new file mode 100644 index 0000000000..1e61c573f2 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaCompareExpression.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class SigmaCompareExpression implements SigmaType { + + public static class CompareOperators { + public static final String LT = "<"; + public static final String LTE = "<="; + public static final String GT = ">"; + public static final String GTE = ">="; + } + + private SigmaFloat number; + + private String op; + + public SigmaCompareExpression(SigmaFloat number, String op) { + this.number = number; + this.op = op; + } + + public SigmaFloat getNumber() { + return number; + } + + public String getOp() { + return op; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaExpansion.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaExpansion.java new file mode 100644 index 0000000000..a80e405c29 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaExpansion.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +import java.util.List; + +public class SigmaExpansion implements SigmaType { + + private List values; + + public SigmaExpansion(List values) { + this.values = values; + } + + public void setValues(List values) { + this.values = values; + } + + public List getValues() { + return values; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaFloat.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaFloat.java new file mode 100644 index 0000000000..307c214d4f --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaFloat.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class SigmaFloat implements SigmaType { + private final Float aFloat; + + public SigmaFloat(Float aFloat) { + this.aFloat = aFloat; + } + + public Float getFloat() { + return aFloat; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaInteger.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaInteger.java new file mode 100644 index 0000000000..9064000a4c --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaInteger.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class SigmaInteger implements SigmaType { + private final Integer integer; + + public SigmaInteger(Integer integer) { + this.integer = integer; + } + + public Integer getInteger() { + return integer; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaNull.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaNull.java new file mode 100644 index 0000000000..a2ac8851ea --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaNull.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public class SigmaNull implements SigmaType { + private Object nullVal = null; + + @Override + public boolean equals(Object o) { + return getClass() == o.getClass(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaRegularExpression.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaRegularExpression.java new file mode 100644 index 0000000000..2abbe1d3d3 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaRegularExpression.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaRegularExpressionError; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SigmaRegularExpression implements SigmaType { + + private String regexp; + + public SigmaRegularExpression(String regexp) throws SigmaRegularExpressionError { + this.regexp = regexp.replace(" ", "_ws_"); + this.compile(); + } + + public void compile() throws SigmaRegularExpressionError { + try { + Pattern.compile(this.regexp); + } catch (Exception ex) { + throw new SigmaRegularExpressionError("Regular expression '" + this.regexp + "' is invalid: " + ex.getMessage()); + } + } + + public String escape(List escaped, String escapeChar) { + if (escapeChar == null || escapeChar.isEmpty()) { + escapeChar = "\\"; + } + + List rList = new ArrayList<>(); + for (String escape: escaped) { + rList.add(Pattern.quote(escape)); + } + rList.add(Pattern.quote(escapeChar)); + String r = String.join("|", rList); + + List pos = new ArrayList<>(); + pos.add(0); + + Pattern pattern = Pattern.compile(r); + Matcher matcher = pattern.matcher(this.regexp); + + while (matcher.find()) { + pos.add(matcher.start()); + } + pos.add(this.regexp.length()); + + List ranges = new ArrayList<>(); + for (int i = 0; i < pos.size()-1; ++i) { + ranges.add(this.regexp.substring(pos.get(i), pos.get(i+1))); + } + return String.join(escapeChar, ranges); + } + + public String getRegexp() { + return regexp; + } + + public void setRegexp(String regexp) { + this.regexp = regexp; + } + + @Override + public String toString() { + return this.regexp; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaString.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaString.java new file mode 100644 index 0000000000..ccbf0dca57 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaString.java @@ -0,0 +1,371 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaValueError; +import org.opensearch.dataprepper.plugins.processor.parser.utils.AnyOneOf; +import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SigmaString implements SigmaType { + + public class SpecialChars { + public static final char WILDCARD_MULTI = '*'; + public static final char WILDCARD_SINGLE = '?'; + public static final char ESCAPE_CHAR = '\\'; + } + + private String original; + + private List> sOpt; + + public SigmaString(String s) { + if (s == null) { + s = ""; + } +// s = s.replace(" ", "_ws_"); + + this.original = s; + int sLen = s.length(); + + List> r = new ArrayList<>(); + StringBuilder acc = new StringBuilder(); + boolean escaped = false; + for (int i = 0; i < sLen; i++) { + if (escaped) { + if (s.charAt(i) == SpecialChars.WILDCARD_MULTI || s.charAt(i) == SpecialChars.WILDCARD_SINGLE + || s.charAt(i) == SpecialChars.ESCAPE_CHAR) { + acc.append(s.charAt(i)); + } else { + acc.append(SpecialChars.ESCAPE_CHAR).append(s.charAt(i)); + } + escaped = false; + } else if (s.charAt(i) == SpecialChars.ESCAPE_CHAR) { + escaped = true; + } else { + if (s.charAt(i) == SpecialChars.WILDCARD_MULTI || s.charAt(i) == SpecialChars.WILDCARD_SINGLE) { + if (!acc.toString().equals("")) { + r.add(AnyOneOf.leftVal(acc.toString())); + } + + switch (s.charAt(i)) { + case SpecialChars.WILDCARD_MULTI: + r.add(AnyOneOf.middleVal(SpecialChars.WILDCARD_MULTI)); + break; + case SpecialChars.WILDCARD_SINGLE: + r.add(AnyOneOf.middleVal(SpecialChars.WILDCARD_SINGLE)); + } + acc = new StringBuilder(); + } else { + acc.append(s.charAt(i)); + } + } + } + + if (escaped) { + acc.append(SpecialChars.ESCAPE_CHAR); + } + if (!acc.toString().equals("")) { + r.add(AnyOneOf.leftVal(acc.toString())); + } + + this.sOpt = r; + } + + public void mergeStrings() { + List> mergedOpts = new ArrayList<>(); + + int size = this.sOpt.size(); + for (int i = 0; i < size; ++i) { + int mSize = mergedOpts.size(); + if (mSize > 0 && mergedOpts.get(mSize-1).isLeft() && this.sOpt.get(i).isLeft()) { + mergedOpts.set(mSize-1, AnyOneOf.leftVal(mergedOpts.get(mSize-1).getLeft() + this.sOpt.get(i).getLeft())); + } else { + mergedOpts.add(this.sOpt.get(i)); + } + } + this.sOpt = mergedOpts; + } + + public SigmaString append(AnyOneOf other) { + this.sOpt.add(other); + this.mergeStrings(); + return this; + } + + public SigmaString prepend(AnyOneOf other) { + this.sOpt.add(0, other); + this.mergeStrings(); + return this; + } + + public int length() { + int sum = 0; + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isLeft()) { + sum += sOptElem.getLeft().length(); + } else { + ++sum; + } + } + return sum; + } + + public boolean startsWith(Either val) { + AnyOneOf c = this.sOpt.get(0); + if (val.isLeft()) { + return c.isLeft() && c.getLeft().startsWith(val.getLeft()); + } else if (val.isRight()) { + return c.isMiddle() && c.getMiddle() == val.get(); + } + return false; + } + + public boolean endsWith(Either val) { + AnyOneOf c = this.sOpt.get(this.sOpt.size()-1); + if (val.isLeft()) { + return c.isLeft() && c.getLeft().endsWith(val.getLeft()); + } else if (val.isRight()) { + return c.isMiddle() && c.getMiddle() == val.get(); + } + return false; + } + + public byte[] getBytes() { + return this.toString().getBytes(Charset.defaultCharset()); + } + + public boolean containsSpecial() { + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isMiddle() && (sOptElem.getMiddle() == SpecialChars.ESCAPE_CHAR || sOptElem.getMiddle() == SpecialChars.WILDCARD_MULTI + || sOptElem.getMiddle() == SpecialChars.WILDCARD_SINGLE)) { + return true; + } + } + return false; + } + + public boolean containsWildcard() { + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isMiddle() && (sOptElem.getMiddle() == SpecialChars.WILDCARD_MULTI + || sOptElem.getMiddle() == SpecialChars.WILDCARD_SINGLE)) { + return true; + } + } + return false; + } + + public String convert(String escapeChar, String wildcardMulti, String wildcardSingle, String addEscaped, String addReserved, String filterChars) throws SigmaValueError { + StringBuilder s = new StringBuilder(); + Set escapedChars = new HashSet<>(); + + if (wildcardMulti != null) { + for (Character c: wildcardMulti.toCharArray()) { + escapedChars.add(c); + } + } + if (wildcardSingle != null) { + for (Character c: wildcardSingle.toCharArray()) { + escapedChars.add(c); + } + } + if (addEscaped != null) { + for (Character c: addEscaped.toCharArray()) { + escapedChars.add(c); + } + } + + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isLeft()) { + if (Arrays.stream(addReserved.split(" ")).anyMatch(s1 -> s1.equals(sOptElem.getLeft()))) { + s.append(escapeChar); + s.append(sOptElem.getLeft()); + } else { + for (Character c : sOptElem.getLeft().toCharArray()) { + if (filterChars.contains(String.valueOf(c))) { + continue; + } + if (escapedChars.contains(c)) { + s.append(escapeChar); + } + s.append(c); + } + } + } else if (sOptElem.getMiddle() != null) { + Character c = sOptElem.getMiddle(); + if (c == SpecialChars.WILDCARD_MULTI) { + if (wildcardMulti != null) { + s.append(wildcardMulti); + } else { + throw new SigmaValueError("Multi-character wildcard not specified for conversion"); + } + } else if (c == SpecialChars.WILDCARD_SINGLE) { + if (wildcardSingle != null) { + s.append(wildcardSingle); + } else { + throw new SigmaValueError("Single-character wildcard not specified for conversion"); + } + } + } + } + return s.toString().replace(" ", "_ws_"); + } + + public SigmaString replaceWithPlaceholder(Pattern regex, String placeholderName) { + List> result = new ArrayList<>(); + + for (AnyOneOf elem: this.sOpt) { + if (elem.isLeft()) { + String elemStr = elem.getLeft(); + boolean matched = false; + + int idx = 0; + Matcher matcher = regex.matcher(elemStr); + while (matcher.find()) { + matched = true; + + String sElem = elemStr.substring(idx, matcher.start()); + if (!sElem.isEmpty()) { + result.add(AnyOneOf.leftVal(sElem)); + } + result.add(AnyOneOf.rightVal(new Placeholder(placeholderName))); + idx = matcher.end(); + } + + if (matched) { + String sElem = elemStr.substring(idx); + if (!sElem.isEmpty()) { + result.add(AnyOneOf.leftVal(sElem)); + } + } else { + result.add(elem); + } + } else { + result.add(elem); + } + } + + SigmaString sStr = new SigmaString(null); + sStr.setsOpt(result); + return sStr; + } + + public boolean containsPlaceholder(List include, List exclude) { + for (AnyOneOf elem: this.sOpt) { + if (elem.isRight() && (include == null || include.contains(elem.get().getName())) && + (exclude == null || !exclude.contains(elem.get().getName()))) { + return true; + } + } + return false; + } + + public List replacePlaceholders(Function>> callback) { + if (!this.containsPlaceholder(null, null)) { + return List.of(this); + } + + List results = new ArrayList<>(); + List> s = this.getsOpt(); + int size = s.size(); + for (int idx = 0; idx < size; ++idx) { + if (s.get(idx).isRight()) { + SigmaString prefix = new SigmaString(null); + + List> presOpt = new ArrayList<>(); + for (int preIdx = 0; preIdx < idx; ++preIdx) { + presOpt.add(s.get(preIdx)); + } + prefix.setsOpt(presOpt); + + Placeholder placeholder = s.get(idx).get(); + + SigmaString suffix = new SigmaString(null); + + List> sufsOpt = new ArrayList<>(); + for (int sufIdx = idx + 1; sufIdx < size; ++sufIdx) { + sufsOpt.add(s.get(sufIdx)); + } + suffix.setsOpt(sufsOpt); + + for (SigmaString resultSuffix: suffix.replacePlaceholders(callback)) { + for (AnyOneOf replacement: callback.apply(placeholder)) { + SigmaString tempSuffix = new SigmaString(null); + tempSuffix.setsOpt(resultSuffix.getsOpt()); + + tempSuffix.prepend(replacement); + prefix.getsOpt().forEach(tempSuffix::prepend); + results.add(tempSuffix); + } + } + return results; + } + } + return results; + } + + public List> getsOpt() { + return sOpt; + } + + public void setsOpt(List> sOpt) { + this.sOpt = new ArrayList<>(); + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isLeft()) { + this.sOpt.add(AnyOneOf.leftVal(sOptElem.getLeft())); + } else if (sOptElem.isMiddle()) { + this.sOpt.add(AnyOneOf.middleVal(sOptElem.getMiddle())); + } else { + this.sOpt.add(AnyOneOf.rightVal(sOptElem.get())); + } + } + } + + public String getOriginal() { + return original; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SigmaString that = (SigmaString) o; + + if (sOpt.size() != that.sOpt.size()) { + return false; + } + + for (int idx = 0; idx < sOpt.size(); ++idx) { + if ((sOpt.get(idx).isLeft() && !that.sOpt.get(idx).isLeft()) || + (sOpt.get(idx).isMiddle() && !that.sOpt.get(idx).isMiddle()) || + (sOpt.get(idx).isRight() && !that.sOpt.get(idx).isRight())) { + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (AnyOneOf sOptElem: sOpt) { + if (sOptElem.isLeft()) { + sb.append(sOptElem.getLeft()); + } else if (sOptElem.isMiddle()) { + sb.append(sOptElem.getMiddle()); + } + } + return sb.toString().replace(" ", "_ws_"); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaType.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaType.java new file mode 100644 index 0000000000..7f897a6e24 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaType.java @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +public interface SigmaType { +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaTypeFacade.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaTypeFacade.java new file mode 100644 index 0000000000..fd349dfe13 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/types/SigmaTypeFacade.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.types; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class SigmaTypeFacade { + + private final Map, Class> typeMap; + private static SigmaTypeFacade typeFacade; + + public SigmaTypeFacade() { + typeMap = new HashMap<>(); + typeMap.put(Boolean.TYPE, SigmaBool.class); + typeMap.put(Integer.TYPE, SigmaInteger.class); + typeMap.put(Float.TYPE, SigmaFloat.class); + typeMap.put(String.class, SigmaString.class); + typeMap.put(Optional.empty().getClass(), SigmaNull.class); + } + + public static SigmaType sigmaType(Object val) { + if (typeFacade == null) { + typeFacade = new SigmaTypeFacade(); + } + + if (val == null) { + return new SigmaNull(); + } else if (val.getClass().equals(Boolean.class)) { + return new SigmaBool((Boolean) val); + } else if (val.getClass().equals(Integer.class)) { + return new SigmaInteger((Integer) val); + } else if (val.getClass().equals(Float.class)) { + return new SigmaFloat((Float) val); + } else if (val.getClass().equals(String.class)) { + return new SigmaString((String) val); + } else { + return new SigmaNull(); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/AnyOneOf.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/AnyOneOf.java new file mode 100644 index 0000000000..40fe5c1388 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/AnyOneOf.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.utils; + +public abstract class AnyOneOf extends Either { + + private static final long serialVersionUID = 1L; + + AnyOneOf() { + super(); + } + + public static AnyOneOf leftVal(L left) { + return new Left<>(left); + } + + public static AnyOneOf rightVal(R right) { + return new Right<>(right); + } + + public static AnyOneOf middleVal(M middle) { + return new Middle<>(middle); + } + + public abstract M getMiddle(); + + public abstract boolean isMiddle(); +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Either.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Either.java new file mode 100644 index 0000000000..8de8315f1d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Either.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.utils; + +import java.io.Serializable; + +public abstract class Either implements Serializable { + + private static final long serialVersionUID = 1L; + + Either() { + } + + public static Either right(R right) { + return new Right<>(right); + } + + public static Either left(L left) { + return new Left<>(left); + } + + public abstract L getLeft(); + + public abstract boolean isLeft(); + + public abstract boolean isRight(); + + public abstract R get(); +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Left.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Left.java new file mode 100644 index 0000000000..72ee71868c --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Left.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.utils; + +import java.util.NoSuchElementException; + +final class Left extends AnyOneOf { + + private static final long serialVersionUID = 1L; + + private final L value; + + public Left(L value) { + this.value = value; + } + + @Override + public L getLeft() { + return value; + } + + @Override + public boolean isLeft() { + return true; + } + + @Override + public boolean isRight() { + return false; + } + + @Override + public R get() { + throw new NoSuchElementException("get() on Left"); + } + + @Override + public M getMiddle() { + throw new NoSuchElementException("getMiddle() on Middle"); + } + + @Override + public boolean isMiddle() { + return false; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Middle.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Middle.java new file mode 100644 index 0000000000..52c3bc770a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Middle.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.utils; + +import java.util.NoSuchElementException; + +final class Middle extends AnyOneOf { + + private static final long serialVersionUID = 1L; + + private final M value; + + public Middle(M value) { + this.value = value; + } + + @Override + public M getMiddle() { + return value; + } + + @Override + public boolean isMiddle() { + return true; + } + + @Override + public L getLeft() { + throw new NoSuchElementException("getLeft() on Middle"); + } + + @Override + public boolean isLeft() { + return false; + } + + @Override + public boolean isRight() { + return false; + } + + @Override + public R get() { + throw new NoSuchElementException("get() on Middle"); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Right.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Right.java new file mode 100644 index 0000000000..950c63291e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/utils/Right.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.utils; + +import java.util.NoSuchElementException; + +final class Right extends AnyOneOf { + + private static final long serialVersionUID = 1L; + + private final R value; + + public Right(R value) { + this.value = value; + } + + @Override + public L getLeft() { + throw new NoSuchElementException("getLeft() on Right"); + } + + @Override + public boolean isLeft() { + return false; + } + + @Override + public boolean isRight() { + return true; + } + + @Override + public R get() { + return value; + } + + @Override + public M getMiddle() { + throw new NoSuchElementException("getMiddle() on Right"); + } + + @Override + public boolean isMiddle() { + return false; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/DetectorDTO.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/DetectorDTO.java new file mode 100644 index 0000000000..eb67a62808 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/DetectorDTO.java @@ -0,0 +1,33 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import java.util.Set; + +public class DetectorDTO { + private Set ruleIds; + private String monitorID; + private String monitorName; + private String findingsIndex; + + public DetectorDTO(final Set ruleIds, final String monitorID, final String monitorName, final String findingsIndex) { + this.ruleIds = ruleIds; + this.monitorID = monitorID; + this.monitorName = monitorName; + this.findingsIndex = findingsIndex; + } + + public Set getRuleIds() { + return ruleIds; + } + + public String getMonitorID() { + return monitorID; + } + + public String getMonitorName() { + return monitorName; + } + + public String getFindingsIndex() { + return findingsIndex; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java new file mode 100644 index 0000000000..be18cc3af4 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java @@ -0,0 +1,69 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import org.opensearch.dataprepper.model.event.Event; + +import java.util.List; +import java.util.function.Predicate; + +public class Rule { + private final String title; + private final String id; + private final Predicate condition; + private final List tags; + private final String query; + private String monitorId; + private String monitorName; + private String findingsIndex; + + public Rule(final String title, final String id, final Predicate condition, final List tags) { + this.title = title; + this.id = id; + this.condition = condition; + this.tags = tags; + this.query = "PLACEHOLDER_QUERY"; + } + + public String getTitle() { + return title; + } + + public String getId() { + return id; + } + + public Predicate getCondition() { + return condition; + } + + public List getTags() { + return tags; + } + + public String getQuery() { + return query; + } + + public String getMonitorId() { + return monitorId; + } + + public String getMonitorName() { + return monitorName; + } + + public String getFindingsIndex() { + return findingsIndex; + } + + public void setMonitorId(final String monitorId) { + this.monitorId = monitorId; + } + + public void setMonitorName(String monitorName) { + this.monitorName = monitorName; + } + + public void setFindingsIndex(final String findingsIndex) { + this.findingsIndex = findingsIndex; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleConverter.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleConverter.java new file mode 100644 index 0000000000..048ff59111 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleConverter.java @@ -0,0 +1,35 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfig; +import org.opensearch.dataprepper.plugins.processor.parser.SigmaRulePredicateParser; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRuleTag; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class RuleConverter implements Function { + private final SigmaRulePredicateParser sigmaRulePredicateParser; + + public RuleConverter(final RuleEngineConfig config) { + this.sigmaRulePredicateParser = new SigmaRulePredicateParser(config); + } + + @Override + public Rule apply(final SigmaRule sigmaRule) { + final Predicate predicate = sigmaRulePredicateParser.parseRule(sigmaRule); + + final List tags = new ArrayList<>(); + tags.add(sigmaRule.getLevel().toString()); + tags.add(sigmaRule.getLogSource().getService()); + sigmaRule.getTags().stream() + .map(SigmaRuleTag::toString) + .forEach(tags::add); + + return new Rule(sigmaRule.getTitle(), sigmaRule.getId().toString(), predicate, tags); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleFetcher.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleFetcher.java new file mode 100644 index 0000000000..e9c3c8fa51 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleFetcher.java @@ -0,0 +1,245 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.MgetRequest; +import org.opensearch.client.opensearch.core.MgetResponse; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.mget.MultiGetResponseItem; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.dataprepper.plugins.processor.exceptions.RuleRefreshException; +import org.opensearch.dataprepper.plugins.processor.model.detector.Detector; +import org.opensearch.dataprepper.plugins.processor.model.detector.DetectorInput; +import org.opensearch.dataprepper.plugins.processor.model.detector.DetectorRule; +import org.opensearch.dataprepper.plugins.processor.model.detector.DetectorWrapper; +import org.opensearch.dataprepper.plugins.processor.model.detector.Input; +import org.opensearch.dataprepper.plugins.processor.model.rule.RuleWrapper; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class RuleFetcher implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(RuleFetcher.class); + + private static final String DETECTORS_INDEX = ".opensearch-sap-detectors-config"; + private static final String PREPACKAGED_RULES_INDEX = ".opensearch-sap-pre-packaged-rules-config"; + private static final String CUSTOM_RULES_INDEX = ".opensearch-sap-custom-rules-config"; + + private final OpenSearchClient openSearchClient; + private final RuleStore sigmaRuleStore; + private final RuleConverter ruleConverter; + + public RuleFetcher(final OpenSearchClient openSearchClient, + final RuleStore sigmaRuleStore, + final RuleConverter ruleConverter) { + this.openSearchClient = openSearchClient; + this.sigmaRuleStore = sigmaRuleStore; + this.ruleConverter = ruleConverter; + } + + @Override + public void run() { + try { + LOG.info("Starting rule fetch"); + final List detectors = getDetectors(); + LOG.info("Found detectors: {}", detectors); + + final Map> indexToPrepackagedRules = getIndexToPrepackagedRules(detectors); + final Map> indexToCustomRules = getIndexToCustomRules(detectors); + + final Map> mergedIndexToRules = mergeIndexToRules(List.of(indexToPrepackagedRules, indexToCustomRules)); + sigmaRuleStore.updateRuleStore(mergedIndexToRules); + } catch (final Exception e) { + LOG.error("Caught exception refreshing rules", e); + } + } + + private List getDetectors() { + final SearchRequest listDetectorsRequest = getListDetectorsRequest(); + + try { + final SearchResponse listDetectorsResponse = openSearchClient.search(listDetectorsRequest, DetectorWrapper.class); + return parseDetectors(listDetectorsResponse); + } catch (final Exception e) { + throw new RuleRefreshException("Exception listing detectors", e); + } + } + + // TODO - build API in OpenSearch + private SearchRequest getListDetectorsRequest() { + return new SearchRequest.Builder() + .index(DETECTORS_INDEX) + .size(10000) // TODO - pagination + .build(); + } + + private List parseDetectors(final SearchResponse listDetectorsResponse) { + return listDetectorsResponse.hits().hits().stream() + .map(Hit::source) + .filter(Objects::nonNull) + .map(DetectorWrapper::getDetector) + .collect(Collectors.toList()); + } + + private Map> getIndexToPrepackagedRules(final List detectors) { + final Map> indexToPrepackagedRuleIds = getIndexToDetectorMetadata(detectors, DetectorInput::getPrePackagedRules); + final Set prepackagedRuleIds = indexToPrepackagedRuleIds.values().stream() + .flatMap(Collection::stream) + .map(DetectorDTO::getRuleIds) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + final Optional getPrepackagedRulesRequest = getMgetRequest(PREPACKAGED_RULES_INDEX, prepackagedRuleIds); + + if (getPrepackagedRulesRequest.isEmpty()) { + // No rules to get + return Collections.emptyMap(); + } + + try { + final MgetResponse getRulesResponse = openSearchClient.mget(getPrepackagedRulesRequest.get(), RuleWrapper.class); + return getIndexToRules(indexToPrepackagedRuleIds, getRulesResponse); + } catch (final Exception e) { + throw new RuleRefreshException("Exception getting prepackaged rules", e); + } + } + + private Map> getIndexToCustomRules(final List detectors) { + final Map> indexToCustomRuleIds = getIndexToDetectorMetadata(detectors, DetectorInput::getCustomRules); + final Set customRuleIds = indexToCustomRuleIds.values().stream() + .flatMap(Collection::stream) + .map(DetectorDTO::getRuleIds) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + final Optional getCustomRulesRequest = getMgetRequest(CUSTOM_RULES_INDEX, customRuleIds); + + if (getCustomRulesRequest.isEmpty()) { + // No rules to get + return Collections.emptyMap(); + } + + try { + final MgetResponse getRulesResponse = openSearchClient.mget(getCustomRulesRequest.get(), RuleWrapper.class); + return getIndexToRules(indexToCustomRuleIds, getRulesResponse); + } catch (final Exception e) { + throw new RuleRefreshException("Exception getting custom rules", e); + } + } + + private Map> getIndexToDetectorMetadata(final List detectors, final Function> ruleGetter) { + final Map> indexToRuleIds = new HashMap<>(); + detectors.stream() + .forEach(detector -> { + final List inputs = detector.getInputs(); + final List detectorInputs = inputs.stream() + .map(Input::getDetectorInput) + .collect(Collectors.toList()); + + detectorInputs.forEach(detectorInput -> { + final Set ruleIds = getRuleIds(detectorInput, ruleGetter); + final DetectorDTO detectorDTO = new DetectorDTO(ruleIds, detector.getMonitorId().get(0), detector.getName(), detector.getFindingsIndex()); + + detectorInput.getIndices().forEach(index -> { + indexToRuleIds.putIfAbsent(index, new ArrayList<>()); + indexToRuleIds.get(index).add(detectorDTO); + }); + }); + }); + + return indexToRuleIds; + } + + private Set getRuleIds(final DetectorInput detectorInput, final Function> ruleGetter) { + return ruleGetter.apply(detectorInput).stream() + .map(DetectorRule::getId) + .collect(Collectors.toSet()); + } + + private Optional getMgetRequest(final String index, final Set docIds) { + if (docIds.isEmpty()) { + return Optional.empty(); + } + + final List docIdsList = new ArrayList<>(docIds); + + return Optional.of(new MgetRequest.Builder() + .index(index) + .ids(docIdsList) + .build()); + } + + private Map> getIndexToRules(final Map> indexToDetectorMetadata, final MgetResponse getRulesResponse) { + final Map ruleIdToSigmaRule = getRuleIdToSigmaRule(getRulesResponse); + + final Map> indexToRules = new HashMap<>(); + indexToDetectorMetadata.forEach((index, detectorDTOs) -> { + final List rules = detectorDTOs.stream() + .map(detectorDTO -> { + final Set ruleIds = detectorDTO.getRuleIds(); + return ruleIds.stream() + .map(ruleId -> { + final Rule rule = ruleIdToSigmaRule.get(ruleId); + rule.setMonitorId(detectorDTO.getMonitorID()); + rule.setFindingsIndex(detectorDTO.getFindingsIndex()); + rule.setMonitorName(detectorDTO.getMonitorName()); + + return rule; + }) + .collect(Collectors.toList()); + }) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + if (!rules.isEmpty()) { + indexToRules.put(index, rules); + } + }); + + return indexToRules; + } + + private Map getRuleIdToSigmaRule(final MgetResponse getRulesResponse) { + return getRulesResponse.docs().stream() + .map(MultiGetResponseItem::result) + .filter(ruleWrapperGetResult -> Objects.nonNull(ruleWrapperGetResult.source())) + .map(ruleWrapperGetResult -> { + final String ruleID = ruleWrapperGetResult.id(); + final Rule sigmaRule = parseSigmaRule(ruleWrapperGetResult.source()); + + return Map.entry(ruleID, sigmaRule); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Rule parseSigmaRule(final RuleWrapper ruleWrapper) { + final String ruleString = ruleWrapper.getRule().getRule(); + final SigmaRule sigmaRule = SigmaRule.fromYaml(ruleString, true); + + return ruleConverter.apply(sigmaRule); + } + + private Map> mergeIndexToRules(final List>> indexToRulesList) { + final Map> indexToRules = new HashMap<>(); + + indexToRulesList.forEach(indexToRulesEntry -> { + indexToRulesEntry.forEach((index, sigmaRules) -> { + indexToRules.putIfAbsent(index, new ArrayList<>()); + indexToRules.get(index).addAll(sigmaRules); + }); + }); + + return indexToRules; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java new file mode 100644 index 0000000000..28036aacbd --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java @@ -0,0 +1,22 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// TODO - does this need locking? +public class RuleStore { + private Map> rules; + + public RuleStore() { + this.rules = new HashMap<>(); + } + + public void updateRuleStore(final Map> updatedRules) { + rules = updatedRules; + } + + public Map> getRules() { + return rules; + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/resources/mappings/cloudtrail.json b/data-prepper-plugins/rule-engine/src/main/resources/mappings/cloudtrail.json new file mode 100644 index 0000000000..cd16f0cb25 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/resources/mappings/cloudtrail.json @@ -0,0 +1,219 @@ +{ + "mappings":[ + { + "raw_field":"eventName", + "ecs":"aws.cloudtrail.event_name", + "ocsf": "api.operation" + }, + { + "raw_field":"eventSource", + "ecs":"aws.cloudtrail.event_source", + "ocsf": "api.service.name" + }, + { + "raw_field":"eventVersion", + "ecs":"aws.cloudtrail.event_version", + "ocsf": "metadata.product.version" + }, + { + "raw_field":"eventID", + "ecs":"aws.cloudtrail.event_id", + "ocsf": "metadata.uid" + }, + { + "raw_field":"eventType", + "ecs":"aws.cloudtrail.event_type", + "ocsf": "unmapped.eventType" + }, + { + "raw_field":"eventCategory", + "ecs":"aws.cloudtrail.event_category", + "ocsf": "metadata.product.feature.name" + }, + { + "raw_field":"errorMessage", + "ecs":"aws.cloudtrail.error_message", + "ocsf": "api.response.message" + }, + { + "raw_field":"errorCode", + "ecs":"aws.cloudtrail.error_code", + "ocsf": "api.response.error" + }, + { + "raw_field":"apiVersion", + "ecs":"aws.cloudtrail.api_version", + "ocsf": "api.version" + }, + { + "raw_field":"awsRegion", + "ecs":"aws.cloudtrail.aws_region", + "ocsf": "cloud.region" + }, + { + "raw_field":"additionalEventData.LoginTo", + "ecs":"aws.cloudtrail.additional_event_data.loginTo", + "ocsf": "dst_endpoint.svc_name" + }, + { + "raw_field":"additionalEventData.MFAUsed", + "ecs":"aws.cloudtrail.additional_event_data.mfaUsed", + "ocsf": "mfa" + }, + { + "raw_field":"responseElements", + "ecs":"aws.cloudtrail.response_elements.text", + "ocsf": "unmapped.responseElements" + }, + { + "raw_field":"requestID", + "ecs":"aws.cloudtrail.request_id", + "ocsf": "api.request.uid" + }, + { + "raw_field":"sourceIPAddress", + "ecs":"aws.cloudtrail.source_ip_address", + "ocsf": "src_endpoint.ip" + }, + { + "raw_field":"userAgent", + "ecs":"aws.cloudtrail.user_agent", + "ocsf": "http_request.user_agent" + }, + { + "raw_field":"vpcEndpointId", + "ecs":"aws.cloudtrail.vpc_endpoint_id", + "ocsf": "src_endpoint.uid" + }, + { + "raw_field":"responseElements.pendingModifiedValues.masterUserPassword", + "ecs":"aws.cloudtrail.response_elements.pending_modified_values.master_user_password", + "ocsf": "unmapped.responseElements.pendingModifiedValues.masterUserPassword" + }, + { + "raw_field":"responseElements.publiclyAccessible", + "ecs":"aws.cloudtrail.response_elements.publicly_accessible", + "ocsf": "unmapped.responseElements.publiclyAccessible" + }, + { + "raw_field":"responseElements.ConsoleLogin", + "ecs":"aws.cloudtrail.response_elements.publicly_accessible", + "ocsf": "status_id" + }, + { + "raw_field":"requestParameters.arn", + "ecs":"aws.cloudtrail.request_parameters.arn", + "ocsf": "unmapped.requestParameters.arn" + }, + { + "raw_field":"requestParameters.attribute", + "ecs":"aws.cloudtrail.request_parameters.attribute", + "ocsf": "unmapped.requestParameters.attribute" + }, + { + "raw_field":"requestParameters.userName", + "ecs":"aws.cloudtrail.request_parameters.username", + "ocsf": "unmapped.requestParameters.userName" + }, + { + "raw_field":"requestParameters.roleArn", + "ecs":"aws.cloudtrail.request_parameters.roleArn", + "ocsf": "user.uuid" + }, + { + "raw_field":"requestParameters.roleSessionName", + "ecs":"aws.cloudtrail.request_parameters.roleSessionName", + "ocsf": "user.name" + }, + { + "raw_field":"requestParameters.containerDefinitions.command", + "ecs":"aws.cloudtrail.request_parameters.container_definitions.command", + "ocsf": "unmapped.requestParameters.containerDefinitions.command" + }, + { + "raw_field":"userIdentity.type", + "ecs":"aws.cloudtrail.user_identity.type", + "ocsf": "actor.user.type" + }, + { + "raw_field":"userIdentity.principalId", + "ecs":"aws.cloudtrail.user_identity.principalId", + "ocsf": "actor.user.uid" + }, + { + "raw_field":"userIdentity.arn", + "ecs":"aws.cloudtrail.user_identity.arn", + "ocsf": "actor.user.uuid" + }, + { + "raw_field":"userIdentity.accountId", + "ecs":"aws.cloudtrail.user_identity.accountId", + "ocsf": "actor.user.account_uid" + }, + { + "raw_field":"userIdentity.accessKeyId", + "ecs":"aws.cloudtrail.user_identity.accessKeyId", + "ocsf": "actor.user.credential_uid" + }, + { + "raw_field":"userIdentity.identityProvider", + "ecs":"aws.cloudtrail.user_identity.identityProvider", + "ocsf": "actor.idp.name" + }, + { + "raw_field":"userIdentity.userName", + "ecs":"aws.cloudtrail.user_identity.userName", + "ocsf": "actor.user.name" + }, + { + "raw_field":"userIdentity.invokedBy", + "ecs":"aws.cloudtrail.user_identity.invokedBy", + "ocsf": "actor.invoked_by" + }, + { + "raw_field":"userIdentity.sessionContext.sessionIssuer.type", + "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.type", + "ocsf": "unmapped.userIdentity.sessionContext.sessionIssuer.type" + }, + { + "raw_field":"userIdentity.sessionContext.sessionIssuer.arn", + "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.arn", + "ocsf": "actor.session.issuer" + }, + { + "raw_field":"userIdentity.sessionContext.attributes.creationDate", + "ecs":"aws.cloudtrail.user_identity.session_context.attributes.creationDate", + "ocsf": "actor.session.created_time" + }, + { + "raw_field":"userIdentity.sessionContext.attributes.mfaAuthenticated", + "ecs":"aws.cloudtrail.user_identity.session_context.attributes.mfaAuthenticated", + "ocsf": "actor.session.mfa" + }, + { + "raw_field":"userIdentity.webIdFederationData.federatedProvider", + "ecs":"aws.cloudtrail.user_identity.web_id_federation_data.federatedProvider", + "ocsf": "actor.idp.name" + }, + { + "raw_field":"resources[].ARN", + "ecs":"aws.cloudtrail.resources.ARN", + "ocsf": "resources[].uid" + }, + { + "raw_field":"resources[].accountId", + "ecs":"aws.cloudtrail.resources.account_uid", + "ocsf": "resources[].account_uid" + }, + { + "raw_field":"resources[].type", + "ecs":"aws.cloudtrail.resources.type", + "ocsf": "resources[].type" + }, + { + "raw_field":"eventTime", + "ecs":"timestamp", + "ocsf": "time" + } + ] +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParserTest.java b/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParserTest.java new file mode 100644 index 0000000000..cb812e1f0f --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaRulePredicateParserTest.java @@ -0,0 +1,114 @@ +package org.opensearch.dataprepper.plugins.processor.parser; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockitoAnnotations; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.log.JacksonLog; +import org.opensearch.dataprepper.plugins.processor.RuleEngineConfig; +import org.opensearch.dataprepper.plugins.processor.model.log.LogFormat; +import org.opensearch.dataprepper.plugins.processor.model.log.LogType; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SigmaRulePredicateParserTest { + private static final String RULES_PATH_FORMAT = "src/test/resources/rules/%s"; + private static final String DELETE_IDENTITY_RULE_FILE = "aws_delete_identity.yml"; + private static final String GUARDDUTY_DISRUPTION_RULE_FILE = "aws_guardduty_disruption.yml"; + + private SigmaRulePredicateParser sigmaRulePredicateParser; + + @BeforeEach + void setup() { + MockitoAnnotations.openMocks(this); + + final RuleEngineConfig ruleEngineConfig = new RuleEngineConfig.Builder() + .withLogFormat(LogFormat.NONE) + .withLogType(LogType.CLOUDTRAIL) + .build(); + sigmaRulePredicateParser = new SigmaRulePredicateParser(ruleEngineConfig); + } + + @Test + void parse_SingleFieldEqualsCondition() { + final SigmaRule sigmaRule = getSigmaRule(DELETE_IDENTITY_RULE_FILE); + final Predicate result = sigmaRulePredicateParser.parseRule(sigmaRule); + + assertTrue(result.test(getEvent(Map.of("eventSource", "ses.amazonaws.com")))); + assertFalse(result.test(getEvent(Map.of("eventSource", UUID.randomUUID().toString())))); + } + + @Test + void parse_SingleFieldEqualsCondition_OCSF() { + sigmaRulePredicateParser = new SigmaRulePredicateParser(new RuleEngineConfig.Builder() + .withLogFormat(LogFormat.OCSF) + .withLogType(LogType.CLOUDTRAIL) + .build()); + + final SigmaRule sigmaRule = getSigmaRule(GUARDDUTY_DISRUPTION_RULE_FILE); + final Predicate result = sigmaRulePredicateParser.parseRule(sigmaRule); + + assertTrue(result.test(getEvent(Map.of( + "api.service.name", "guardduty.amazonaws.com", + "api.operation", "CreateIPSet" + )))); + assertFalse(result.test(getEvent(Map.of( + "api.service.name", "guardduty.amazonaws.com", + "api.operation", "DeleteIPSet" + )))); + } + + @Test + void parse_SingleFieldEqualsCondition_Nested() { + sigmaRulePredicateParser = new SigmaRulePredicateParser(new RuleEngineConfig.Builder() + .withLogFormat(LogFormat.OCSF) + .withLogType(LogType.CLOUDTRAIL) + .build()); + + final SigmaRule sigmaRule = getSigmaRule(GUARDDUTY_DISRUPTION_RULE_FILE); + final Predicate result = sigmaRulePredicateParser.parseRule(sigmaRule); + + assertTrue(result.test(getEvent(Map.of( + "api", Map.of( + "service", Map.of( + "name", "guardduty.amazonaws.com" + ), + "operation", "CreateIPSet" + ) + )))); + assertFalse(result.test(getEvent(Map.of( + "api", Map.of( + "service", Map.of( + "name", "guardduty.amazonaws.com" + ), + "operation", "DeleteIPSet" + ) + )))); + } + + private SigmaRule getSigmaRule(final String ruleFile) { + try { + final Path rulePath = Path.of(String.format(RULES_PATH_FORMAT, ruleFile)); + final String ruleString = Files.readString(rulePath, StandardCharsets.UTF_8); + + return SigmaRule.fromYaml(ruleString, true); + } catch (final Exception e) { + throw new RuntimeException("Exception parsing rule: " + ruleFile, e); + } + } + + private Event getEvent(final Map data) { + return JacksonLog.builder() + .withData(data) + .build(); + } +} diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_attached_malicious_lambda_layer.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_attached_malicious_lambda_layer.yml new file mode 100644 index 0000000000..783dceb324 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_attached_malicious_lambda_layer.yml @@ -0,0 +1,25 @@ +title: AWS Attached Malicious Lambda Layer +id: 97fbabf8-8e1b-47a2-b7d5-a418d2b95e3d +status: test +description: | + Detects when an user attached a Lambda layer to an existing function to override a library that is in use by the function, where their malicious code could utilize the function's IAM role for AWS API calls. + This would give an adversary access to the privileges associated with the Lambda service role that is attached to that function. +references: + - https://docs.aws.amazon.com/lambda/latest/dg/API_UpdateFunctionConfiguration.html +author: Austin Songer +date: 2021/09/23 +modified: 2022/10/09 +tags: + - attack.privilege_escalation +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: lambda.amazonaws.com + eventName|startswith: 'UpdateFunctionConfiguration' + condition: selection +falsepositives: + - Lambda Layer being attached may be performed by a system administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - Lambda Layer being attached from unfamiliar users should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_cloudtrail_disable_logging.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_cloudtrail_disable_logging.yml new file mode 100644 index 0000000000..eeae3dc7ac --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_cloudtrail_disable_logging.yml @@ -0,0 +1,26 @@ +title: AWS CloudTrail Important Change +id: 4db60cc0-36fb-42b7-9b58-a5b53019fb74 +status: test +description: Detects disabling, deleting and updating of a Trail +references: + - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/best-practices-security.html +author: vitaliy0x1 +date: 2020/01/21 +modified: 2022/10/09 +tags: + - attack.defense_evasion + - attack.t1562.001 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: cloudtrail.amazonaws.com + eventName: + - StopLogging + - UpdateTrail + - DeleteTrail + condition: selection_source +falsepositives: + - Valid change in a Trail +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_config_disable_recording.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_config_disable_recording.yml new file mode 100644 index 0000000000..ea3d2e7b33 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_config_disable_recording.yml @@ -0,0 +1,25 @@ +title: AWS Config Disabling Channel/Recorder +id: 07330162-dba1-4746-8121-a9647d49d297 +status: test +description: Detects AWS Config Service disabling +references: + - https://docs.aws.amazon.com/config/latest/developerguide/cloudtrail-log-files-for-aws-config.html +author: vitaliy0x1 +date: 2020/01/21 +modified: 2022/10/09 +tags: + - attack.defense_evasion + - attack.t1562.001 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'config.amazonaws.com' + eventName: + - 'DeleteDeliveryChannel' + - 'StopConfigurationRecorder' + condition: selection +falsepositives: + - Valid change in AWS Config Service +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_delete_identity.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_delete_identity.yml new file mode 100644 index 0000000000..69f3ae41a7 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_delete_identity.yml @@ -0,0 +1,22 @@ +title: SES Identity Has Been Deleted +id: 20f754db-d025-4a8f-9d74-e0037e999a9a +status: test +description: Detects an instance of an SES identity being deleted via the "DeleteIdentity" event. This may be an indicator of an adversary removing the account that carried out suspicious or malicious activities +references: + - https://unit42.paloaltonetworks.com/compromised-cloud-compute-credentials/ +author: Janantha Marasinghe +date: 2022/12/13 +modified: 2022/12/28 +tags: + - attack.defense_evasion + - attack.t1070 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'ses.amazonaws.com' + condition: selection +falsepositives: + - Unknown +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_disable_bucket_versioning.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_disable_bucket_versioning.yml new file mode 100644 index 0000000000..e3694277be --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_disable_bucket_versioning.yml @@ -0,0 +1,23 @@ +title: AWS S3 Bucket Versioning Disable +id: a136ac98-b2bc-4189-a14d-f0d0388e57a7 +status: experimental +description: Detects when S3 bucket versioning is disabled. Threat actors use this technique during AWS ransomware incidents prior to deleting S3 objects. +references: + - https://invictus-ir.medium.com/ransomware-in-the-cloud-7f14805bbe82 +author: Sean Johnstone | Unit 42 +date: 2023/10/28 +tags: + - attack.impact + - attack.t1490 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: s3.amazonaws.com + eventName: PutBucketVersioning + requestParameters|contains: 'Suspended' + condition: selection +falsepositives: + - AWS administrator legitimately disabling bucket versioning +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_disable_encryption.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_disable_encryption.yml new file mode 100644 index 0000000000..ff2d9cd140 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_disable_encryption.yml @@ -0,0 +1,27 @@ +title: AWS EC2 Disable EBS Encryption +id: 16124c2d-e40b-4fcc-8f2c-5ab7870a2223 +status: stable +description: | + Identifies disabling of default Amazon Elastic Block Store (EBS) encryption in the current region. + Disabling default encryption does not change the encryption status of your existing volumes. +references: + - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DisableEbsEncryptionByDefault.html +author: Sittikorn S +date: 2021/06/29 +modified: 2021/08/20 +tags: + - attack.impact + - attack.t1486 + - attack.t1565 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: ec2.amazonaws.com + eventName: DisableEbsEncryptionByDefault + condition: selection +falsepositives: + - System Administrator Activities + - DEV, UAT, SAT environment. You should apply this rule with PROD account only. +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_startup_script_change.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_startup_script_change.yml new file mode 100644 index 0000000000..8a17154543 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_startup_script_change.yml @@ -0,0 +1,26 @@ +title: AWS EC2 Startup Shell Script Change +id: 1ab3c5ed-5baf-417b-bb6b-78ca33f6c3df +status: test +description: Detects changes to the EC2 instance startup script. The shell script will be executed as root/SYSTEM every time the specific instances are booted up. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/ec2__startup_shell_script/main.py#L9 +author: faloker +date: 2020/02/12 +modified: 2022/06/07 +tags: + - attack.execution + - attack.t1059.001 + - attack.t1059.003 + - attack.t1059.004 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: ec2.amazonaws.com + requestParameters.attribute: 'userData' + eventName: ModifyInstanceAttribute + condition: selection_source +falsepositives: + - Valid changes to the startup script +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_vm_export_failure.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_vm_export_failure.yml new file mode 100644 index 0000000000..7b4a0e5da6 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ec2_vm_export_failure.yml @@ -0,0 +1,29 @@ +title: AWS EC2 VM Export Failure +id: 54b9a76a-3c71-4673-b4b3-2edb4566ea7b +status: test +description: An attempt to export an AWS EC2 instance has been detected. A VM Export might indicate an attempt to extract information from an instance. +references: + - https://docs.aws.amazon.com/vm-import/latest/userguide/vmexport.html#export-instance +author: Diogo Braz +date: 2020/04/16 +modified: 2022/10/05 +tags: + - attack.collection + - attack.t1005 + - attack.exfiltration + - attack.t1537 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventName: 'CreateInstanceExportTask' + eventSource: 'ec2.amazonaws.com' + filter1: + errorMessage|contains: '*' + filter2: + errorCode|contains: '*' + filter3: + responseElements|contains: 'Failure' + condition: selection and not 1 of filter* +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ecs_task_definition_cred_endpoint_query.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ecs_task_definition_cred_endpoint_query.yml new file mode 100644 index 0000000000..257b7f6266 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_ecs_task_definition_cred_endpoint_query.yml @@ -0,0 +1,31 @@ +title: AWS ECS Task Definition That Queries The Credential Endpoint +id: b94bf91e-c2bf-4047-9c43-c6810f43baad +status: experimental +description: | + Detects when an Elastic Container Service (ECS) Task Definition includes a command to query the credential endpoint. + This can indicate a potential adversary adding a backdoor to establish persistence or escalate privileges. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/ecs__backdoor_task_def/main.py + - https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html + - https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +author: Darin Smith +date: 2022/06/07 +modified: 2023/04/24 +tags: + - attack.persistence + - attack.t1525 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'ecs.amazonaws.com' + eventName: + - 'DescribeTaskDefinition' + - 'RegisterTaskDefinition' + - 'RunTask' + requestParameters.containerDefinitions.command|contains: '$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI' + condition: selection +falsepositives: + - Task Definition being modified to request credentials from the Task Metadata Service for valid reasons +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_modified_or_deleted.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_modified_or_deleted.yml new file mode 100644 index 0000000000..d4df9c3cf2 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_modified_or_deleted.yml @@ -0,0 +1,25 @@ +title: AWS EFS Fileshare Modified or Deleted +id: 25cb1ba1-8a19-4a23-a198-d252664c8cef +status: test +description: | + Detects when a EFS Fileshare is modified or deleted. + You can't delete a file system that is in use. + If the file system has any mount targets, the adversary must first delete them, so deletion of a mount will occur before deletion of a fileshare. +references: + - https://docs.aws.amazon.com/efs/latest/ug/API_DeleteFileSystem.html +author: Austin Songer @austinsonger +date: 2021/08/15 +modified: 2022/10/09 +tags: + - attack.impact +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: elasticfilesystem.amazonaws.com + eventName: DeleteFileSystem + condition: selection +falsepositives: + - Unknown +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_mount_modified_or_deleted.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_mount_modified_or_deleted.yml new file mode 100644 index 0000000000..da66ea29aa --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_efs_fileshare_mount_modified_or_deleted.yml @@ -0,0 +1,23 @@ +title: AWS EFS Fileshare Mount Modified or Deleted +id: 6a7ba45c-63d8-473e-9736-2eaabff79964 +status: test +description: Detects when a EFS Fileshare Mount is modified or deleted. An adversary breaking any file system using the mount target that is being deleted, which might disrupt instances or applications using those mounts. +references: + - https://docs.aws.amazon.com/efs/latest/ug/API_DeleteMountTarget.html +author: Austin Songer @austinsonger +date: 2021/08/15 +modified: 2022/10/09 +tags: + - attack.impact + - attack.t1485 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: elasticfilesystem.amazonaws.com + eventName: DeleteMountTarget + condition: selection +falsepositives: + - Unknown +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_eks_cluster_created_or_deleted.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_eks_cluster_created_or_deleted.yml new file mode 100644 index 0000000000..2418354757 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_eks_cluster_created_or_deleted.yml @@ -0,0 +1,27 @@ +title: AWS EKS Cluster Created or Deleted +id: 33d50d03-20ec-4b74-a74e-1e65a38af1c0 +status: test +description: Identifies when an EKS cluster is created or deleted. +references: + - https://any-api.com/amazonaws_com/eks/docs/API_Description +author: Austin Songer +date: 2021/08/16 +modified: 2022/10/09 +tags: + - attack.impact + - attack.t1485 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: eks.amazonaws.com + eventName: + - CreateCluster + - DeleteCluster + condition: selection +falsepositives: + - EKS Cluster being created or deleted may be performed by a system administrator. + - Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - EKS Cluster created or deleted from unfamiliar users should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_created.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_created.yml new file mode 100644 index 0000000000..415f69cb17 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_created.yml @@ -0,0 +1,26 @@ +title: AWS ElastiCache Security Group Created +id: 4ae68615-866f-4304-b24b-ba048dfa5ca7 +status: test +description: Detects when an ElastiCache security group has been created. +references: + - https://github.com/elastic/detection-rules/blob/598f3d7e0a63221c0703ad9a0ea7e22e7bc5961e/rules/integrations/aws/persistence_elasticache_security_group_creation.toml +author: Austin Songer @austinsonger +date: 2021/07/24 +modified: 2022/10/09 +tags: + - attack.persistence + - attack.t1136 + - attack.t1136.003 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: elasticache.amazonaws.com + eventName: 'CreateCacheSecurityGroup' + condition: selection +falsepositives: + - A ElastiCache security group may be created by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Security group creations from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. + + +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_modified_or_deleted.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_modified_or_deleted.yml new file mode 100644 index 0000000000..8c162d317e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_elasticache_security_group_modified_or_deleted.yml @@ -0,0 +1,30 @@ +title: AWS ElastiCache Security Group Modified or Deleted +id: 7c797da2-9cf2-4523-ba64-33b06339f0cc +status: test +description: Identifies when an ElastiCache security group has been modified or deleted. +references: + - https://github.com/elastic/detection-rules/blob/7d5efd68603f42be5e125b5a6a503b2ef3ac0f4e/rules/integrations/aws/impact_elasticache_security_group_modified_or_deleted.toml +author: Austin Songer @austinsonger +date: 2021/07/24 +modified: 2022/10/09 +tags: + - attack.impact + - attack.t1531 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: elasticache.amazonaws.com + eventName: + - 'DeleteCacheSecurityGroup' + - 'AuthorizeCacheSecurityGroupIngress' + - 'RevokeCacheSecurityGroupIngress' + - 'AuthorizeCacheSecurityGroupEgress' + - 'RevokeCacheSecurityGroupEgress' + condition: selection +falsepositives: + - A ElastiCache security group deletion may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Security Group deletions from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. + + +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_enum_buckets.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_enum_buckets.yml new file mode 100644 index 0000000000..ae7f84a06a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_enum_buckets.yml @@ -0,0 +1,30 @@ +title: Potential Bucket Enumeration on AWS +id: f305fd62-beca-47da-ad95-7690a0620084 +related: + - id: 4723218f-2048-41f6-bcb0-417f2d784f61 + type: similar +status: experimental +description: Looks for potential enumeration of AWS buckets via ListBuckets. +references: + - https://github.com/Lifka/hacking-resources/blob/c2ae355d381bd0c9f0b32c4ead049f44e5b1573f/cloud-hacking-cheat-sheets.md + - https://jamesonhacking.blogspot.com/2020/12/pivoting-to-private-aws-s3-buckets.html + - https://securitycafe.ro/2022/12/14/aws-enumeration-part-ii-practical-enumeration/ +author: Christopher Peacock @securepeacock, SCYTHE @scythe_io +date: 2023/01/06 +modified: 2023/04/28 +tags: + - attack.discovery + - attack.t1580 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 's3.amazonaws.com' + eventName: 'ListBuckets' + filter: + type: 'AssumedRole' + condition: selection and not filter +falsepositives: + - Administrators listing buckets, it may be necessary to filter out users who commonly conduct this activity. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_guardduty_disruption.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_guardduty_disruption.yml new file mode 100644 index 0000000000..b81f188c40 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_guardduty_disruption.yml @@ -0,0 +1,23 @@ +title: AWS GuardDuty Important Change +id: 6e61ee20-ce00-4f8d-8aee-bedd8216f7e3 +status: test +description: Detects updates of the GuardDuty list of trusted IPs, perhaps to disable security alerts against malicious IPs. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/guardduty__whitelist_ip/main.py#L9 +author: faloker +date: 2020/02/11 +modified: 2022/10/09 +tags: + - attack.defense_evasion + - attack.t1562.001 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: guardduty.amazonaws.com + eventName: CreateIPSet + condition: selection_source +falsepositives: + - Valid change in the GuardDuty (e.g. to ignore internal scanners) +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_backdoor_users_keys.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_backdoor_users_keys.yml new file mode 100644 index 0000000000..085d768cfa --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_backdoor_users_keys.yml @@ -0,0 +1,34 @@ +title: AWS IAM Backdoor Users Keys +id: 0a5177f4-6ca9-44c2-aacf-d3f3d8b6e4d2 +status: test +description: | + Detects AWS API key creation for a user by another user. + Backdoored users can be used to obtain persistence in the AWS environment. + Also with this alert, you can detect a flow of AWS keys in your org. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/iam__backdoor_users_keys/main.py +author: faloker +date: 2020/02/12 +modified: 2022/10/09 +tags: + - attack.persistence + - attack.t1098 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: iam.amazonaws.com + eventName: CreateAccessKey + filter: + userIdentity.arn|contains: responseElements.accessKey.userName + condition: selection_source and not filter +fields: + - userIdentity.arn + - responseElements.accessKey.userName + - errorCode + - errorMessage +falsepositives: + - Adding user keys to their own accounts (the filter cannot cover all possible variants of user naming) + - AWS API keys legitimate exchange workflows +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_loginprofile_creation.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_loginprofile_creation.yml new file mode 100644 index 0000000000..6755f3547a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_loginprofile_creation.yml @@ -0,0 +1,27 @@ +title: AWS IAM S3Browser LoginProfile Creation +id: db014773-b1d3-46bd-ba26-133337c0ffee +status: experimental +description: Detects S3 Browser utility performing reconnaissance looking for existing IAM Users without a LoginProfile defined then (when found) creating a LoginProfile. +references: + - https://permiso.io/blog/s/unmasking-guivil-new-cloud-threat-actor +author: daniel.bohannon@permiso.io (@danielhbohannon) +date: 2023/05/17 +tags: + - attack.execution + - attack.persistence + - attack.t1059.009 + - attack.t1078.004 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'iam.amazonaws.com' + eventName: + - 'GetLoginProfile' + - 'CreateLoginProfile' + userAgent|contains: 'S3 Browser' + condition: selection +falsepositives: + - Valid usage of S3 Browser for IAM LoginProfile listing and/or creation +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_templated_s3_bucket_policy_creation.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_templated_s3_bucket_policy_creation.yml new file mode 100644 index 0000000000..3f38039a20 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_templated_s3_bucket_policy_creation.yml @@ -0,0 +1,30 @@ +title: AWS IAM S3Browser Templated S3 Bucket Policy Creation +id: db014773-7375-4f4e-b83b-133337c0ffee +status: experimental +description: Detects S3 browser utility creating Inline IAM policy containing default S3 bucket name placeholder value of "". +references: + - https://permiso.io/blog/s/unmasking-guivil-new-cloud-threat-actor +author: daniel.bohannon@permiso.io (@danielhbohannon) +date: 2023/05/17 +modified: 2023/05/17 +tags: + - attack.execution + - attack.t1059.009 + - attack.persistence + - attack.t1078.004 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: iam.amazonaws.com + eventName: PutUserPolicy + userAgent|contains: 'S3 Browser' + requestParameters|contains|all: + - '"arn:aws:s3:::/*"' + - '"s3:GetObject"' + - '"Allow"' + condition: selection +falsepositives: + - Valid usage of S3 browser with accidental creation of default Inline IAM policy without changing default S3 bucket name placeholder value +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_user_or_accesskey_creation.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_user_or_accesskey_creation.yml new file mode 100644 index 0000000000..e4e9323a4d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_iam_s3browser_user_or_accesskey_creation.yml @@ -0,0 +1,27 @@ +title: AWS IAM S3Browser User or AccessKey Creation +id: db014773-d9d9-4792-91e5-133337c0ffee +status: experimental +description: Detects S3 Browser utility creating IAM User or AccessKey. +references: + - https://permiso.io/blog/s/unmasking-guivil-new-cloud-threat-actor +author: daniel.bohannon@permiso.io (@danielhbohannon) +date: 2023/05/17 +tags: + - attack.execution + - attack.persistence + - attack.t1059.009 + - attack.t1078.004 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'iam.amazonaws.com' + eventName: + - 'CreateUser' + - 'CreateAccessKey' + userAgent|contains: 'S3 Browser' + condition: selection +falsepositives: + - Valid usage of S3 Browser for IAM User and/or AccessKey creation +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_passed_role_to_glue_development_endpoint.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_passed_role_to_glue_development_endpoint.yml new file mode 100644 index 0000000000..f7b05b387e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_passed_role_to_glue_development_endpoint.yml @@ -0,0 +1,27 @@ +title: AWS Glue Development Endpoint Activity +id: 4990c2e3-f4b8-45e3-bc3c-30b14ff0ed26 +status: test +description: Detects possible suspicious glue development endpoint activity. +references: + - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation/ + - https://docs.aws.amazon.com/glue/latest/webapi/API_CreateDevEndpoint.html +author: Austin Songer @austinsonger +date: 2021/10/03 +modified: 2022/12/18 +tags: + - attack.privilege_escalation +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: 'glue.amazonaws.com' + eventName: + - 'CreateDevEndpoint' + - 'DeleteDevEndpoint' + - 'UpdateDevEndpoint' + condition: selection +falsepositives: + - Glue Development Endpoint Activity may be performed by a system administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_change_master_password.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_change_master_password.yml new file mode 100644 index 0000000000..c3b990d9df --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_change_master_password.yml @@ -0,0 +1,24 @@ +title: AWS RDS Master Password Change +id: 8a63cdd4-6207-414a-85bc-7e032bd3c1a2 +status: test +description: Detects the change of database master password. It may be a part of data exfiltration. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/rds__explore_snapshots/main.py +author: faloker +date: 2020/02/12 +modified: 2022/10/05 +tags: + - attack.exfiltration + - attack.t1020 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: rds.amazonaws.com + responseElements.pendingModifiedValues.masterUserPassword|contains: '*' + eventName: ModifyDBInstance + condition: selection_source +falsepositives: + - Benign changes to a db instance +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_public_db_restore.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_public_db_restore.yml new file mode 100644 index 0000000000..597a66a6f0 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_rds_public_db_restore.yml @@ -0,0 +1,24 @@ +title: Restore Public AWS RDS Instance +id: c3f265c7-ff03-4056-8ab2-d486227b4599 +status: test +description: Detects the recovery of a new public database instance from a snapshot. It may be a part of data exfiltration. +references: + - https://github.com/RhinoSecurityLabs/pacu/blob/866376cd711666c775bbfcde0524c817f2c5b181/pacu/modules/rds__explore_snapshots/main.py +author: faloker +date: 2020/02/12 +modified: 2022/10/09 +tags: + - attack.exfiltration + - attack.t1020 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: rds.amazonaws.com + responseElements.publiclyAccessible: 'true' + eventName: RestoreDBInstanceFromDBSnapshot + condition: selection_source +falsepositives: + - Unknown +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_root_account_usage.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_root_account_usage.yml new file mode 100644 index 0000000000..5470622d77 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_root_account_usage.yml @@ -0,0 +1,24 @@ +title: AWS Root Credentials +id: 8ad1600d-e9dc-4251-b0ee-a65268f29add +status: test +description: Detects AWS root account usage +references: + - https://docs.aws.amazon.com/IAM/latest/UserGuide/id_root-user.html +author: vitaliy0x1 +date: 2020/01/21 +modified: 2022/10/09 +tags: + - attack.privilege_escalation + - attack.t1078.004 +logsource: + product: aws + service: cloudtrail +detection: + selection_usertype: + userIdentity.type: Root + selection_eventtype: + eventType: AwsServiceEvent + condition: selection_usertype and not selection_eventtype +falsepositives: + - AWS Tasks That Require AWS Account Root User Credentials https://docs.aws.amazon.com/general/latest/gr/aws_tasks-that-require-root.html +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_lock_disabled.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_lock_disabled.yml new file mode 100644 index 0000000000..bf738eff0b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_lock_disabled.yml @@ -0,0 +1,26 @@ +title: AWS Route 53 Domain Transfer Lock Disabled +id: 3940b5f1-3f46-44aa-b746-ebe615b879e0 +status: test +description: Detects when a transfer lock was removed from a Route 53 domain. It is recommended to refrain from performing this action unless intending to transfer the domain to a different registrar. +references: + - https://github.com/elastic/detection-rules/blob/c76a39796972ecde44cb1da6df47f1b6562c9770/rules/integrations/aws/persistence_route_53_domain_transfer_lock_disabled.toml + - https://docs.aws.amazon.com/Route53/latest/APIReference/API_Operations_Amazon_Route_53.html + - https://docs.aws.amazon.com/Route53/latest/APIReference/API_domains_DisableDomainTransferLock.html +author: Elastic, Austin Songer @austinsonger +date: 2021/07/22 +modified: 2022/10/09 +tags: + - attack.persistence + - attack.credential_access + - attack.t1098 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: route53.amazonaws.com + eventName: DisableDomainTransferLock + condition: selection +falsepositives: + - A domain transfer lock may be disabled by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Activity from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_to_another_account.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_to_another_account.yml new file mode 100644 index 0000000000..599badbcdc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_route_53_domain_transferred_to_another_account.yml @@ -0,0 +1,24 @@ +title: AWS Route 53 Domain Transferred to Another Account +id: b056de1a-6e6e-4e40-a67e-97c9808cf41b +status: test +description: Detects when a request has been made to transfer a Route 53 domain to another AWS account. +references: + - https://github.com/elastic/detection-rules/blob/c76a39796972ecde44cb1da6df47f1b6562c9770/rules/integrations/aws/persistence_route_53_domain_transferred_to_another_account.toml +author: Elastic, Austin Songer @austinsonger +date: 2021/07/22 +modified: 2022/10/09 +tags: + - attack.persistence + - attack.credential_access + - attack.t1098 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: route53.amazonaws.com + eventName: TransferDomainToAnotherAwsAccount + condition: selection +falsepositives: + - A domain may be transferred to another AWS account by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. Domain transfers from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_s3_data_management_tampering.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_s3_data_management_tampering.yml new file mode 100644 index 0000000000..393dbbc73b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_s3_data_management_tampering.yml @@ -0,0 +1,36 @@ +title: AWS S3 Data Management Tampering +id: 78b3756a-7804-4ef7-8555-7b9024a02e2d +status: test +description: Detects when a user tampers with S3 data management in Amazon Web Services. +references: + - https://github.com/elastic/detection-rules/pull/1145/files + - https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations.html + - https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLogging.html + - https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketWebsite.html + - https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html + - https://docs.aws.amazon.com/AmazonS3/latest/userguide/setting-repl-config-perm-overview.html + - https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html +author: Austin Songer @austinsonger +date: 2021/07/24 +modified: 2022/10/09 +tags: + - attack.exfiltration + - attack.t1537 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: s3.amazonaws.com + eventName: + - PutBucketLogging + - PutBucketWebsite + - PutEncryptionConfiguration + - PutLifecycleConfiguration + - PutReplicationConfiguration + - ReplicateObject + - RestoreObject + condition: selection +falsepositives: + - A S3 configuration change may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. S3 configuration change from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_securityhub_finding_evasion.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_securityhub_finding_evasion.yml new file mode 100644 index 0000000000..a32ca64866 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_securityhub_finding_evasion.yml @@ -0,0 +1,30 @@ +title: AWS SecurityHub Findings Evasion +id: a607e1fe-74bf-4440-a3ec-b059b9103157 +status: stable +description: Detects the modification of the findings on SecurityHub. +references: + - https://docs.aws.amazon.com/cli/latest/reference/securityhub/ +author: Sittikorn S +date: 2021/06/28 +tags: + - attack.defense_evasion + - attack.t1562 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: securityhub.amazonaws.com + eventName: + - 'BatchUpdateFindings' + - 'DeleteInsight' + - 'UpdateFindings' + - 'UpdateInsight' + condition: selection +fields: + - sourceIPAddress + - userIdentity.arn +falsepositives: + - System or Network administrator behaviors + - DEV, UAT, SAT environment. You should apply this rule with PROD environment only. +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_snapshot_backup_exfiltration.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_snapshot_backup_exfiltration.yml new file mode 100644 index 0000000000..ced4493a53 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_snapshot_backup_exfiltration.yml @@ -0,0 +1,23 @@ +title: AWS Snapshot Backup Exfiltration +id: abae8fec-57bd-4f87-aff6-6e3db989843d +status: test +description: Detects the modification of an EC2 snapshot's permissions to enable access from another account +references: + - https://www.justice.gov/file/1080281/download +author: Darin Smith +date: 2021/05/17 +modified: 2021/08/19 +tags: + - attack.exfiltration + - attack.t1537 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: ec2.amazonaws.com + eventName: ModifySnapshotAttribute + condition: selection_source +falsepositives: + - Valid change to a snapshot's permissions +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sso_idp_change.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sso_idp_change.yml new file mode 100644 index 0000000000..b299af75d8 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sso_idp_change.yml @@ -0,0 +1,32 @@ +title: AWS Identity Center Identity Provider Change +id: d3adb3ef-b7e7-4003-9092-1924c797db35 +status: experimental +description: | + Detects a change in the AWS Identity Center (FKA AWS SSO) identity provider. + A change in identity provider allows an attacker to establish persistent access or escalate privileges via user impersonation. +references: + - https://docs.aws.amazon.com/singlesignon/latest/userguide/app-enablement.html + - https://docs.aws.amazon.com/singlesignon/latest/userguide/sso-info-in-cloudtrail.html + - https://docs.aws.amazon.com/service-authorization/latest/reference/list_awsiamidentitycentersuccessortoawssinglesign-on.html +author: Michael McIntyre @wtfender +date: 2023/09/27 +tags: + - attack.persistence + - attack.t1556 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: + - 'sso-directory.amazonaws.com' + - 'sso.amazonaws.com' + eventName: + - 'AssociateDirectory' + - 'DisableExternalIdPConfigurationForDirectory' + - 'DisassociateDirectory' + - 'EnableExternalIdPConfigurationForDirectory' + condition: selection +falsepositives: + - Authorized changes to the AWS account's identity provider +level: high diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_assumerole_misuse.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_assumerole_misuse.yml new file mode 100644 index 0000000000..bc0615dcfc --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_assumerole_misuse.yml @@ -0,0 +1,29 @@ +title: AWS STS AssumeRole Misuse +id: 905d389b-b853-46d0-9d3d-dea0d3a3cd49 +status: test +description: Identifies the suspicious use of AssumeRole. Attackers could move laterally and escalate privileges. +references: + - https://github.com/elastic/detection-rules/pull/1214 + - https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html +author: Austin Songer @austinsonger +date: 2021/07/24 +modified: 2022/10/09 +tags: + - attack.lateral_movement + - attack.privilege_escalation + - attack.t1548 + - attack.t1550 + - attack.t1550.001 +logsource: + product: aws + service: cloudtrail +detection: + selection: + userIdentity.type: AssumedRole + userIdentity.sessionContext.sessionIssuer.type: Role + condition: selection +falsepositives: + - AssumeRole may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - AssumeRole from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. + - Automated processes that uses Terraform may lead to false positives. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_getsessiontoken_misuse.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_getsessiontoken_misuse.yml new file mode 100644 index 0000000000..817c97a064 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_sts_getsessiontoken_misuse.yml @@ -0,0 +1,28 @@ +title: AWS STS GetSessionToken Misuse +id: b45ab1d2-712f-4f01-a751-df3826969807 +status: test +description: Identifies the suspicious use of GetSessionToken. Tokens could be created and used by attackers to move laterally and escalate privileges. +references: + - https://github.com/elastic/detection-rules/pull/1213 + - https://docs.aws.amazon.com/STS/latest/APIReference/API_GetSessionToken.html +author: Austin Songer @austinsonger +date: 2021/07/24 +modified: 2022/10/09 +tags: + - attack.lateral_movement + - attack.privilege_escalation + - attack.t1548 + - attack.t1550 + - attack.t1550.001 +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: sts.amazonaws.com + eventName: GetSessionToken + userIdentity.type: IAMUser + condition: selection +falsepositives: + - GetSessionToken may be done by a system or network administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. GetSessionToken from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: low diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_susp_saml_activity.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_susp_saml_activity.yml new file mode 100644 index 0000000000..531596e17d --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_susp_saml_activity.yml @@ -0,0 +1,34 @@ +title: AWS Suspicious SAML Activity +id: f43f5d2f-3f2a-4cc8-b1af-81fde7dbaf0e +status: test +description: Identifies when suspicious SAML activity has occurred in AWS. An adversary could gain backdoor access via SAML. +references: + - https://docs.aws.amazon.com/IAM/latest/APIReference/API_UpdateSAMLProvider.html + - https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html +author: Austin Songer +date: 2021/09/22 +modified: 2022/12/18 +tags: + - attack.initial_access + - attack.t1078 + - attack.lateral_movement + - attack.t1548 + - attack.privilege_escalation + - attack.t1550 + - attack.t1550.001 +logsource: + product: aws + service: cloudtrail +detection: + selection_sts: + eventSource: 'sts.amazonaws.com' + eventName: 'AssumeRoleWithSAML' + selection_iam: + eventSource: 'iam.amazonaws.com' + eventName: 'UpdateSAMLProvider' + condition: 1 of selection_* +falsepositives: + - Automated processes that uses Terraform may lead to false positives. + - SAML Provider could be updated by a system administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - SAML Provider being updated from unfamiliar users should be investigated. If known behavior is causing false positives, it can be exempted from the rule. +level: medium diff --git a/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_update_login_profile.yml b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_update_login_profile.yml new file mode 100644 index 0000000000..32f2cd365f --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/test/resources/rules/aws_update_login_profile.yml @@ -0,0 +1,32 @@ +title: AWS User Login Profile Was Modified +id: 055fb148-60f8-462d-ad16-26926ce050f1 +status: test +description: | + An attacker with the iam:UpdateLoginProfile permission on other users can change the password used to login to the AWS console on any user that already has a login profile setup. + With this alert, it is used to detect anyone is changing password on behalf of other users. +references: + - https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation +author: toffeebr33k +date: 2021/08/09 +modified: 2022/10/09 +tags: + - attack.persistence + - attack.t1098 +logsource: + product: aws + service: cloudtrail +detection: + selection_source: + eventSource: iam.amazonaws.com + eventName: UpdateLoginProfile + filter: + userIdentity.arn|contains: requestParameters.userName + condition: selection_source and not filter +fields: + - userIdentity.arn + - requestParameters.userName + - errorCode + - errorMessage +falsepositives: + - Legit User Account Administration +level: high diff --git a/settings.gradle b/settings.gradle index d1d77074ee..8fff0a0866 100644 --- a/settings.gradle +++ b/settings.gradle @@ -163,3 +163,4 @@ include 'data-prepper-plugins:buffer-common' include 'data-prepper-plugins:dissect-processor' include 'data-prepper-plugins:dynamodb-source' include 'data-prepper-plugins:decompress-processor' +include 'data-prepper-plugins:rule-engine'