From 656a5fecbc2e09212b2d621c86b525fcbf9e4086 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 7 Mar 2024 17:24:57 -0800 Subject: [PATCH] Add an "exists" check for "not" condition in sigma rules (#852) * test design Signed-off-by: Joanne Wang * working version Signed-off-by: Joanne Wang * cleaning up Signed-off-by: Joanne Wang * testing Signed-off-by: Joanne Wang * working version Signed-off-by: Joanne Wang * working version Signed-off-by: Joanne Wang * refactored querybackend Signed-off-by: Joanne Wang * working on tests Signed-off-by: Joanne Wang * fixed alerting and finding tests Signed-off-by: Joanne Wang * fix correlation tests Signed-off-by: Joanne Wang * working all tests Signed-off-by: Joanne Wang * moved test and changed alias for adldap Signed-off-by: Joanne Wang * added more tests Signed-off-by: Joanne Wang * cleanup code Signed-off-by: Joanne Wang * remove exists flag Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 142 ++++++--- .../rules/backend/QueryBackend.java | 102 +++--- .../securityanalytics/TestHelpers.java | 147 ++++++++- .../securityanalytics/alerts/AlertsIT.java | 3 +- .../CorrelationEngineRestApiIT.java | 2 +- .../securityanalytics/findings/FindingIT.java | 298 +++++++++++++++++- .../rules/backend/QueryBackendTests.java | 174 +++++++++- 7 files changed, 779 insertions(+), 89 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index ec7b09505..50d452f6b 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -48,7 +48,6 @@ import java.util.Map; public class OSQueryBackend extends QueryBackend { - private String tokenSeparator; private String orToken; @@ -57,6 +56,8 @@ public class OSQueryBackend extends QueryBackend { private String notToken; + private String existsToken; + private String escapeChar; private String wildcardMulti; @@ -119,6 +120,7 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.orToken = "OR"; this.andToken = "AND"; this.notToken = "NOT"; + this.existsToken = "_exists_"; this.escapeChar = "\\"; this.wildcardMulti = "*"; this.wildcardSingle = "?"; @@ -145,15 +147,15 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, } @Override - public Object convertConditionAsInExpression(Either condition) { + public Object convertConditionAsInExpression(Either condition, boolean isConditionNot, boolean applyDeMorgans) { if (condition.isLeft()) { - return this.convertConditionAnd(condition.getLeft()); + return this.convertConditionAnd(condition.getLeft(), isConditionNot, applyDeMorgans); } - return this.convertConditionOr(condition.get()); + return this.convertConditionOr(condition.get(), isConditionNot, applyDeMorgans); } @Override - public Object convertConditionAnd(ConditionAND condition) { + public Object convertConditionAnd(ConditionAND condition, boolean isConditionNot, boolean applyDeMorgans) { try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -171,21 +173,29 @@ public Object convertConditionAnd(ConditionAND condition) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType); + converted = this.convertConditionGroup(argType, isConditionNot,applyDeMorgans ); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isConditionNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isConditionNot, applyDeMorgans); } if (converted != null) { + // if applyDeMorgans is true, then use OR instead of AND + if (applyDeMorgans) { + joiner.setLength(0); // clear the joiner to convert it to OR + if (this.tokenSeparator.equals(this.andToken)) { + joiner.append(this.orToken); + } else { + joiner.append(this.tokenSeparator).append(this.orToken).append(this.tokenSeparator); + } + } if (!first) { queryBuilder.append(joiner).append(converted); } else { queryBuilder.append(converted); first = false; } - } } } @@ -196,7 +206,7 @@ public Object convertConditionAnd(ConditionAND condition) { } @Override - public Object convertConditionOr(ConditionOR condition) { + public Object convertConditionOr(ConditionOR condition, boolean isConditionNot, boolean applyDeMorgans) { try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -214,32 +224,41 @@ public Object convertConditionOr(ConditionOR condition) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType); + converted = this.convertConditionGroup(argType, isConditionNot, applyDeMorgans); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isConditionNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isConditionNot, applyDeMorgans); } if (converted != null) { + // if applyDeMorgans is true, then use AND instead of OR + if (applyDeMorgans) { + joiner.setLength(0); // clear the joiner to convert it to AND + if (this.tokenSeparator.equals(this.orToken)) { + joiner.append(this.andToken); + } else { + joiner.append(this.tokenSeparator).append(this.andToken).append(this.tokenSeparator); + } + } + if (!first) { queryBuilder.append(joiner).append(converted); } else { queryBuilder.append(converted); first = false; } - } } } return queryBuilder.toString(); } catch (Exception ex) { - throw new NotImplementedException("Operator 'and' not supported by the backend"); + throw new NotImplementedException("Operator 'or' not supported by the backend"); } } @Override - public Object convertConditionNot(ConditionNOT condition) { + public Object convertConditionNot(ConditionNOT condition, boolean isConditionNot, boolean applyDeMorgans) { Either, String> arg = condition.getArgs().get(0); try { if (arg.isLeft()) { @@ -247,13 +266,13 @@ public Object convertConditionNot(ConditionNOT condition) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class) ? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))) : (arg.getLeft().getLeft().getClass().equals(ConditionOR.class) ? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))) : new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertConditionGroup(argType)); + return String.format(Locale.getDefault(), groupExpression, this.convertConditionGroup(argType, true, true)); } else if (arg.getLeft().isMiddle()) { ConditionType argType = new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); } else { ConditionType argType = new ConditionType(Either.right(Either.right(arg.getLeft().get()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); } } } catch (Exception ex) { @@ -263,56 +282,89 @@ public Object convertConditionNot(ConditionNOT condition) { } @Override - public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { + public Object convertExistsField(ConditionFieldEqualsValueExpression condition) { + String field = getFinalField(condition.getField()); + return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + this.tokenSeparator + field); + } + + @Override + public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); String expr = "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); + String exprWithDeMorgansApplied = this.notToken + " " + "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); - return String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); + String convertedExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); + if (applyDeMorgans) { + convertedExpr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, this.convertValueStr(value)); + } + return convertedExpr; } @Override - public Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); SigmaNumber number = (SigmaNumber) condition.getValue(); ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); - + if (applyDeMorgans) { + return this.notToken + " " +field + this.eqToken + " " + condition.getValue(); + } return field + this.eqToken + " " + condition.getValue(); } @Override - public Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Collections.singletonMap("type", "boolean")); - + if (applyDeMorgans) { + return this.notToken + " " + field + this.eqToken + " " + ((SigmaBool) condition.getValue()).isaBoolean(); + } return field + this.eqToken + " " + ((SigmaBool) condition.getValue()).isaBoolean(); } - public Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.fieldNullExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field); + } return String.format(Locale.getDefault(), this.fieldNullExpression, field); } @Override - public Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.reExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, convertValueRe((SigmaRegularExpression) condition.getValue())); + } return String.format(Locale.getDefault(), this.reExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue())); } @Override - public Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.cidrExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, convertValueCidr((SigmaCIDRExpression) condition.getValue())); + } return String.format(Locale.getDefault(), this.cidrExpression, field, convertValueCidr((SigmaCIDRExpression) condition.getValue())); } @Override - public Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + this.compareOpExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.getMappedField(condition.getField()), + compareOperators.get(((SigmaCompareExpression) condition.getValue()).getOp()), ((SigmaCompareExpression) condition.getValue()).getNumber().toString()); + } + return String.format(Locale.getDefault(), this.compareOpExpression, this.getMappedField(condition.getField()), compareOperators.get(((SigmaCompareExpression) condition.getValue()).getOp()), ((SigmaCompareExpression) condition.getValue()).getNumber().toString()); } @@ -330,23 +382,39 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre }*/ @Override - public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { + public Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { String field = getFinalValueField(); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); - return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + + String conditionValStr = String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); + if (applyDeMorgans) { + conditionValStr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValStr); + } + return conditionValStr; } @Override - public Object convertConditionValNum(ConditionValueExpression condition) { - return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString()); + public Object convertConditionValNum(ConditionValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + String conditionValNum = String.format(Locale.getDefault(), String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString())); + if (applyDeMorgans) { + conditionValNum = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValNum); + } + return conditionValNum; } @Override - public Object convertConditionValRe(ConditionValueExpression condition) { - return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); + public Object convertConditionValRe(ConditionValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + String conditionValStr = String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); + if (applyDeMorgans) { + conditionValStr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValStr); + } + return conditionValStr; } // TODO: below methods will be supported when Sigma Expand Modifier is supported. @@ -421,8 +489,8 @@ private boolean comparePrecedence(ConditionType outer, ConditionType inner) { return idxInner <= precedence.indexOf(outerClass); } - private Object convertConditionGroup(ConditionType condition) throws SigmaValueError { - return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition)); + private Object convertConditionGroup(ConditionType condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition, isConditionNot, applyDeMorgans)); } private Object convertValueStr(SigmaString s) throws SigmaValueError { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index c63dce05d..2c56a2c6a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -4,8 +4,6 @@ */ package org.opensearch.securityanalytics.rules.backend; -import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder; -import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend.AggregationQueries; import org.opensearch.securityanalytics.rules.condition.ConditionAND; @@ -47,7 +45,6 @@ import java.util.Set; public abstract class QueryBackend { - private boolean convertOrAsIn; private boolean convertAndAsIn; private boolean collectErrors; @@ -85,15 +82,15 @@ public List convertRule(SigmaRule rule) throws SigmaError { Object query; if (conditionItem instanceof ConditionAND) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem))), false, false); } else if (conditionItem instanceof ConditionOR) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem))), false, false); } else if (conditionItem instanceof ConditionNOT) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem))), true, false); } else if (conditionItem instanceof ConditionFieldEqualsValueExpression) { - query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem))), false, false); } else { - query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); if (aggItem != null) { @@ -113,30 +110,41 @@ public List convertRule(SigmaRule rule) throws SigmaError { return queries; } - public Object convertCondition(ConditionType conditionType) throws SigmaValueError { + public Object convertCondition(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { if (conditionType.isConditionOR()) { if (this.decideConvertConditionAsInExpression(Either.right(conditionType.getConditionOR()))) { - return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR())); + return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()), isConditionNot, applyDeMorgans ); } else { - return this.convertConditionOr(conditionType.getConditionOR()); + return this.convertConditionOr(conditionType.getConditionOR(), isConditionNot, applyDeMorgans); } } else if (conditionType.isConditionAND()) { if (this.decideConvertConditionAsInExpression(Either.left(conditionType.getConditionAND()))) { - return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND())); + return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()), isConditionNot, applyDeMorgans); } else { - return this.convertConditionAnd(conditionType.getConditionAND()); + return this.convertConditionAnd(conditionType.getConditionAND(), isConditionNot, applyDeMorgans); } } else if (conditionType.isConditionNOT()) { - return this.convertConditionNot(conditionType.getConditionNOT()); + return this.convertConditionNot(conditionType.getConditionNOT(), isConditionNot, applyDeMorgans); } else if (conditionType.isEqualsValueExpression()) { - return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression()); + // check to see if conditionNot is an ancestor of the parse tree, otherwise return as normal + if (isConditionNot) { + return this.convertConditionFieldEqValNot(conditionType, isConditionNot, applyDeMorgans); + } else { + return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); + } } else if (conditionType.isValueExpression()) { - return this.convertConditionVal(conditionType.getValueExpression()); + return this.convertConditionVal(conditionType.getValueExpression(), applyDeMorgans); } else { throw new IllegalArgumentException("Unexpected data type in condition parse tree"); } } + public String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); + String addExists = this.convertExistsField(conditionType.getEqualsValueExpression()).toString(); + return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); + } + public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { return false; @@ -181,74 +189,76 @@ public void resetQueryFields() { } } - public abstract Object convertConditionAsInExpression(Either condition); + public abstract Object convertConditionAsInExpression(Either condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionAnd(ConditionAND condition); + public abstract Object convertConditionAnd(ConditionAND condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionOr(ConditionOR condition); + public abstract Object convertConditionOr(ConditionOR condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionNot(ConditionNOT condition); + public abstract Object convertConditionNot(ConditionNOT condition, boolean isConditionNot, boolean applyDeMorgans); - public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { + public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionFieldEqValStr(condition); + return this.convertConditionFieldEqValStr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { - return this.convertConditionFieldEqValNum(condition); + return this.convertConditionFieldEqValNum(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaBool) { - return this.convertConditionFieldEqValBool(condition); + return this.convertConditionFieldEqValBool(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaRegularExpression) { - return this.convertConditionFieldEqValRe(condition); + return this.convertConditionFieldEqValRe(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaCIDRExpression) { - return this.convertConditionFieldEqValCidr(condition); + return this.convertConditionFieldEqValCidr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaCompareExpression) { - return this.convertConditionFieldEqValOpVal(condition); + return this.convertConditionFieldEqValOpVal(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNull) { - return this.convertConditionFieldEqValNull(condition); + return this.convertConditionFieldEqValNull(condition, applyDeMorgans); }/* TODO: below methods will be supported when Sigma Expand Modifier is supported. else if (condition.getValue() instanceof SigmaQueryExpression) { return this.convertConditionFieldEqValQueryExpr(condition); }*/ else if (condition.getValue() instanceof SigmaExpansion) { - return this.convertConditionFieldEqValQueryExpansion(condition); + return this.convertConditionFieldEqValQueryExpansion(condition, isConditionNot, applyDeMorgans); } else { throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName()); } } - public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError; + public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError; + + public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition); + public abstract Object convertExistsField(ConditionFieldEqualsValueExpression condition); -/* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ + /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ - public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) { List, String>> args = new ArrayList<>(); for (SigmaType sigmaType: ((SigmaExpansion) condition.getValue()).getValues()) { args.add(Either.left(AnyOneOf.middleVal(new ConditionFieldEqualsValueExpression(condition.getField(), sigmaType)))); } ConditionOR conditionOR = new ConditionOR(false, args); - return this.convertConditionOr(conditionOR); + return this.convertConditionOr(conditionOR, isConditionNot, applyDeMorgans); } - public Object convertConditionVal(ConditionValueExpression condition) throws SigmaValueError { + public Object convertConditionVal(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionValStr(condition); + return this.convertConditionValStr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { - return this.convertConditionValNum(condition); + return this.convertConditionValNum(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaBool) { throw new SigmaValueError("Boolean values can't appear as standalone value without a field name."); } else if (condition.getValue() instanceof SigmaRegularExpression) { - return this.convertConditionValRe(condition); + return this.convertConditionValRe(condition, applyDeMorgans); }/* else if (condition.getValue() instanceof SigmaCIDRExpression) { throw new SigmaValueError("CIDR values can't appear as standalone value without a field name."); } else if (condition.getValue() instanceof SigmaQueryExpression) { @@ -258,11 +268,11 @@ public Object convertConditionVal(ConditionValueExpression condition) throws Sig } } - public abstract Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError; + public abstract Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError; - public abstract Object convertConditionValNum(ConditionValueExpression condition); + public abstract Object convertConditionValNum(ConditionValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionValRe(ConditionValueExpression condition); + public abstract Object convertConditionValRe(ConditionValueExpression condition, boolean applyDeMorgans); /* public abstract Object convertConditionValQueryExpr(ConditionValueExpression condition);*/ diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d907b797c..2902dbaa7 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -259,6 +259,72 @@ public static String randomRule() { "level: high"; } + public static String randomRuleWithNotCondition() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " AccountType: TestAccountType\n" + + " selection2:\n" + + " AccountName: TestAccountName\n" + + " selection3:\n" + + " EventID: 22\n" + + " condition: (not selection1 and not selection2) and selection3\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithNotConditionBoolAndNum() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " Initiated: \"false\"\n" + + " selection2:\n" + + " AccountName: TestAccountName\n" + + " selection3:\n" + + " EventID: 21\n" + + " condition: not selection1 and not selection3\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomNullRule() { return "title: null field\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -1701,6 +1767,44 @@ public static String randomDoc(int severity, int version, String opCode) { } + public static String randomDocForNotCondition(int severity, int version, String opCode) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"%s\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + + } + public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + @@ -1840,6 +1944,46 @@ public static String randomDoc() { "}"; } + public static String randomNetworkDoc() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"id.orig_h\": \"123.12.123.12\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { return "{\n" + " \"AccountName\": \"" + accountId + "\",\n" + @@ -1857,6 +2001,7 @@ public static String randomVpcFlowDoc() { " \"srcport\": 9000,\n" + " \"dstport\": 8000,\n" + " \"severity_id\": \"-1\",\n" + + " \"id.orig_h\": \"1.2.3.4\",\n" + " \"class_name\": \"Network Activity\"\n" + "}"; } @@ -2432,7 +2577,7 @@ public static List randomLowerCaseStringList() { stringList.add(randomLowerCaseString()); return stringList; } - + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index fbd091595..347fb66f1 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -43,6 +43,7 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; +import static org.opensearch.securityanalytics.TestHelpers.randomNetworkDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; @@ -545,7 +546,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); indexDoc(index1, "1", randomDoc()); - indexDoc(index2, "1", randomDoc()); + indexDoc(index2, "1", randomNetworkDoc()); // execute monitor 1 Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index 149f8fd34..a4cdb6d1c 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -1109,7 +1109,7 @@ private String createAdLdapDetector(String indexName) throws IOException { " \"partial\": true,\n" + " \"alias_mappings\": {\n" + " \"properties\": {\n" + - " \"azure-signinlogs-properties-user_id\": {\n" + + " \"azure.signinlogs.properties.user_id\": {\n" + " \"path\": \"azure.signinlogs.props.user_id\",\n" + " \"type\": \"alias\"\n" + " },\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 3b7ca3c0a..1f7d112de 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -10,29 +10,42 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.ArrayList; +import java.util.Arrays; import java.util.stream.Collectors; import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithNotConditionBoolAndNum; +import static org.opensearch.securityanalytics.TestHelpers.randomNetworkDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocForNotCondition; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithNotCondition; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; @@ -234,7 +247,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); indexDoc(index1, "1", randomDoc()); - indexDoc(index2, "1", randomDoc()); + indexDoc(index2, "1", randomNetworkDoc()); // execute monitor 1 Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -400,6 +413,289 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter restoreAlertsFindingsIMSettings(); } + public void testCreateDetectorWithNotCondition_verifyFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithNotCondition()); + List prepackagedRules = getRandomPrePackagedRules(); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(6, response.getHits().getTotalHits().value); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify 5 prepackaged rules and 1 custom rule + assertEquals(6, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + Set docRuleIds = new HashSet<>(prepackagedRules); + docRuleIds.add(randomDocRuleId); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Collectors.toSet()); + + assertTrue(docRuleIds.containsAll(aggRulesFinding)); + + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + + public void testCreateDetectorWithNotCondition_verifyFindings_success_boolAndNum() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random custom doc rule with NOT condition + String randomDocRuleId = createRule(randomRuleWithNotConditionBoolAndNum()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); + + assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(2, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + + // Verify 1 custom rule + assertEquals(1, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + + /* + Create a detector with custom rules that include a "not" condition in the sigma rule. + Insert two test documents one matching the rule and one without the field matching the condition to generate only one finding + */ + public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random custom doc rule with NOT condition + String randomDocRuleId = createRule(randomRuleWithNotCondition()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); + + assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDocForNotCondition(2, 5, "Test")); + indexDoc(index, "3", randomDocForNotCondition(2, 5, "Test")); + indexDoc(index, "4", randomDoc(2, 5, "Test")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + + // Verify 1 custom rule + assertEquals(1, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "4").containsAll(foundDocIds)); + } + public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1"); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 3f8196d3d..1ec872f88 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -714,7 +714,177 @@ public void testConvertNot() throws IOException, SigmaError { " sel:\n" + " fieldA: value1\n" + " condition: not sel", false)); - Assert.assertEquals("(NOT fieldA: \"value1\")", queries.get(0).toString()); + Assert.assertEquals("(NOT fieldA: \"value1\" AND _exists_: fieldA)", queries.get(0).toString()); + } + + public void testConvertNotWithParenthesis() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " Opcode: Info\n" + + " sel2:\n" + + " Severity: value2\n" + + " condition: not (sel1 or sel2)", false)); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); + } + + public void testConvertNotComplicatedExpression() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection1:\n" + + " CommandLine|endswith: '.cpl'\n" + + " filter:\n" + + " CommandLine|contains:\n" + + " - '\\System32\\'\n" + + " - '%System%'\n" + + " fp1_igfx:\n" + + " CommandLine|contains|all:\n" + + " - 'regsvr32 '\n" + + " - ' /s '\n" + + " - 'igfxCPL.cpl'\n" + + " condition: selection1 and not filter and not fp1_igfx", false)); + Assert.assertEquals("((CommandLine: *.cpl) AND ((((NOT CommandLine: *\\\\System32\\\\* AND _exists_: CommandLine) AND " + + "(NOT CommandLine: *%System%* AND _exists_: CommandLine))))) AND ((((NOT CommandLine: *regsvr32_ws_* AND _exists_: CommandLine) OR " + + "(NOT CommandLine: *_ws_\\/s_ws_* AND _exists_: CommandLine) OR (NOT CommandLine: *igfxCPL.cpl* AND _exists_: CommandLine))))", queries.get(0).toString()); + } + + public void testConvertNotWithAnd() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection:\n" + + " EventType: SetValue\n" + + " TargetObject|endswith: '\\Software\\Microsoft\\WAB\\DLLPath'\n" + + " filter:\n" + + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + + " condition: selection and not filter", false)); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: Details))", queries.get(0).toString()); + } + + public void testConvertNotWithOrAndList() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " field1: valueA1\n" + + " field2: valueA2\n" + + " field3: valueA3\n" + + " sel3:\n" + + " - resp_mime_types|contains: 'dosexec'\n" + + " - c-uri|endswith: '.exe'\n" + + " condition: not sel1 or sel3", false)); + Assert.assertEquals("((((NOT field1: \"valueA1\" AND _exists_: field1) OR (NOT field2: \"valueA2\" AND _exists_: field2) OR (NOT field3: \"valueA3\" AND _exists_: field3)))) OR ((resp_mime_types: *dosexec*) OR (c-uri: *.exe))", queries.get(0).toString()); + } + + public void testConvertNotWithNumAndBool() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " field1: 1\n" + + " sel2:\n" + + " field2: true\n" + + " condition: not sel1 and not sel2", false)); + Assert.assertEquals("((NOT field1: 1 AND _exists_: field1)) AND ((NOT field2: true AND _exists_: field2))", queries.get(0).toString()); + } + + public void testConvertNotWithNull() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: null\n" + + " sel2:\n" + + " fieldB: true\n" + + " condition: not sel1", false)); + Assert.assertEquals("(NOT fieldA: (NOT [* TO *]) AND _exists_: fieldA)", queries.get(0).toString()); + } + + public void testConvertNotWithKeywords() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldB: value2\n" + + " keywords:\n" + + " - test1\n" + + " - 123\n" + + " condition: not keywords", false)); + Assert.assertEquals("(((NOT \"test1\") AND (NOT \"123\")))", queries.get(0).toString()); } public void testConvertPrecedence() throws IOException, SigmaError { @@ -740,7 +910,7 @@ public void testConvertPrecedence() throws IOException, SigmaError { " sel4:\n" + " fieldD: value5\n" + " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((NOT ((fieldC: \"value4\") AND (fieldD: \"value5\"))))", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: fieldC) OR (NOT fieldD: \"value5\" AND _exists_: fieldD))))", queries.get(0).toString()); } public void testConvertMultiConditions() throws IOException, SigmaError {