From 323989db7ef8bc5f0f618bfbaabeb5acdbdb13ba Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 16 Jan 2024 14:42:14 -0800 Subject: [PATCH] support object fields in aggregation based sigma rules (#789) Signed-off-by: Subhobrata Dey --- .../aggregation/AggregationLexer.java | 88 ++++---- src/main/grammars/Aggregation.g4 | 2 +- .../rules/backend/AggregationBuilders.java | 2 +- .../rules/backend/OSQueryBackend.java | 27 ++- .../securityanalytics/TestHelpers.java | 203 ++++++++++++++++++ .../resthandler/DetectorMonitorRestApiIT.java | 200 ++++++++++++++++- .../aggregation/AggregationBackendTests.java | 88 +++++++- 7 files changed, 542 insertions(+), 68 deletions(-) diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java index fcf7ae82a..115766bb1 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) { "\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+ "\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+ "\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+ - "\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+ - "\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+ - "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+ - "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+ - "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+ - "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+ - "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+ - "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+ - "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+ - "\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+ - "&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+ - "\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+ - "\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+ - "\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+ - "\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+ - "\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+ - ">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+ - "=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+ - "\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+ - "\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+ - "\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+ - "u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+ - "\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+ - "\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+ - "\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+ - "\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+ - "\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+ - "C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+ - "F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+ - "\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+ - "-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+ - "\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+ - "PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+ - "\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+ - "\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+ - "\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+ - "\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+ - "\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+ - "\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+ - "\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+ - "\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+ - "\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+ - "\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+ - "\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; + "\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+ + "z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+ + "\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+ + "\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+ + "\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+ + "\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+ + "\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+ + "\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+ + "\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+ + "\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+ + "\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+ + "\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+ + "\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+ + "@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+ + "\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+ + "\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+ + "\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+ + "$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+ + "\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+ + "*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+ + "-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+ + "01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+ + "\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+ + "\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+ + "i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+ + "m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+ + "\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+ + "\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+ + "\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+ + "\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+ + "KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+ + "MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+ + "\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+ + "\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+ + "\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+ + "\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+ + "YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+ + "\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+ + "\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+ + "_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+ + "\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+ + "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+ + "fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+ + "\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/src/main/grammars/Aggregation.g4 b/src/main/grammars/Aggregation.g4 index 00303b68a..be395c5ae 100644 --- a/src/main/grammars/Aggregation.g4 +++ b/src/main/grammars/Aggregation.g4 @@ -21,7 +21,7 @@ RPAREN : ')' ; DECIMAL : '-'?[0-9]+('.'[0-9]+)? ; -IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ; +IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ; WS : [ \r\t\u000C\n]+ -> skip ; comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java index 3927186fb..c0f6bbb7a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat aggregationBuilder = new TermsAggregationBuilder(name).field(name); break; case "count": - aggregationBuilder = new ValueCountAggregationBuilder(name).field(name); + aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name); break; default: throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction)); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index bf8b883ff..ddbeb3b79 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -361,14 +361,15 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { BucketSelectorExtAggregationBuilder condition; String bucketTriggerSelectorId = UUIDs.base64UUID(); - if (aggregation.getAggFunction().equals("count")) { + if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) { String fieldName; - if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) { + if (aggregation.getGroupByField() == null) { fieldName = "_index"; fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index"); } else { - fieldName = aggregation.getGroupByField(); - fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField()); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fieldName = mappedGroupByField; + fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField); } aggBuilder.field(fieldName); fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_cnt", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()); @@ -376,17 +377,23 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold())); condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_cnt"), script, "result_agg", null); } else { - fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField()); - fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()); + /** + * removing dots to eliminate dots in aggregation names + */ + String mappedAggField = getFinalField(aggregation.getAggField()); + String mappedAggFieldUpdated = mappedAggField.replace(".", "_"); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField); + fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()); // Add subaggregation - AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField()); + AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField); if (subAgg != null) { - aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg); + aggBuilder.field(mappedGroupByField).subAggregation(subAgg); } - Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold())); - condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null); + Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold())); + condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null); } AggregationQueries aggregationQueries = new AggregationQueries(); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d6a866baf..19cd6b0b1 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -786,6 +786,109 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + + public static String randomCloudtrailAggrRuleWithDotFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String randomCloudtrailAggrRuleWithEcsFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" + + " selection1:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String cloudtrailOcsfMappings() { + return "\"properties\": {\n" + + " \"time\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"cloud.region\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\"type\": \"keyword\"},\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"text\"}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -1651,6 +1754,106 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailOcsfDoc() { + return "{\n" + + " \"activity_id\": 8,\n" + + " \"activity_name\": \"Detach Policy\",\n" + + " \"actor\": {\n" + + " \"idp\": {\n" + + " \"name\": null\n" + + " },\n" + + " \"invoked_by\": null,\n" + + " \"session\": {\n" + + " \"created_time\": 1702510696000,\n" + + " \"issuer\": \"arn\",\n" + + " \"mfa\": false\n" + + " },\n" + + " \"user\": {\n" + + " \"account_uid\": \"\",\n" + + " \"credential_uid\": \"\",\n" + + " \"name\": null,\n" + + " \"type\": \"AssumedRole\",\n" + + " \"uid\": \"\",\n" + + " \"uuid\": \"\"\n" + + " }\n" + + " },\n" + + " \"api\": {\n" + + " \"operation\": \"CreateFunction\",\n" + + " \"request\": {\n" + + " \"uid\": \"0966237c-6279-43f4-a9d7-1eb416fca17d\"\n" + + " },\n" + + " \"response\": {\n" + + " \"error\": null,\n" + + " \"message\": null\n" + + " },\n" + + " \"service\": {\n" + + " \"name\": \"lambda.amazonaws.com\"\n" + + " },\n" + + " \"version\": null\n" + + " },\n" + + " \"category_name\": \"Audit Activity\",\n" + + " \"category_uid\": 3,\n" + + " \"class_name\": \"account_change\",\n" + + " \"class_uid\": 3001,\n" + + " \"cloud\": {\n" + + " \"provider\": \"AWS\",\n" + + " \"region\": \"us-east-1\"\n" + + " },\n" + + " \"dst_endpoint\": null,\n" + + " \"http_request\": {\n" + + " \"user_agent\": \"Boto3/1.26.90 Python/3.7.17 Linux/test.amzn2.x86_64 exec-env/AWS_Lambda_python3.7 Botocore/1.29.90\"\n" + + " },\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"feature\": {\n" + + " \"name\": \"Management\"\n" + + " },\n" + + " \"name\": \"cloudtrail\",\n" + + " \"vendor_name\": \"AWS\",\n" + + " \"version\": \"1.08\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\"\n" + + " ],\n" + + " \"uid\": \"\",\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"mfa\": null,\n" + + " \"resources\": null,\n" + + " \"severity\": \"Informational\",\n" + + " \"severity_id\": 1,\n" + + " \"src_endpoint\": {\n" + + " \"domain\": null,\n" + + " \"ip\": \"\",\n" + + " \"uid\": null\n" + + " },\n" + + " \"status\": \"Success\",\n" + + " \"status_id\": 1,\n" + + " \"time\": 1702952105000,\n" + + " \"type_name\": \"Account Change: Detach Policy\",\n" + + " \"type_uid\": 300108,\n" + + " \"unmapped\": {\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": \"true\",\n" + + " \"readOnly\": \"false\",\n" + + " \"recipientAccountId\": \"\",\n" + + " \"requestParameters.instanceProfileName\": \"\",\n" + + " \"tlsDetails.cipherSuite\": \"\",\n" + + " \"tlsDetails.clientProvidedHostHeader\": \"iam.amazonaws.com\",\n" + + " \"tlsDetails.tlsVersion\": \"TLSv1.2\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.accountId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.principalId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.type\": \"Role\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.userName\": \"\"\n" + + " },\n" + + " \"user\": {\n" + + " \"name\": \"\",\n" + + " \"uid\": null,\n" + + " \"uuid\": null\n" + + " }\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 47e517f3c..ac90a5d39 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -50,6 +50,44 @@ import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +import static org.opensearch.securityanalytics.TestHelpers.cloudtrailOcsfMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; + public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { /** * 1. Creates detector with 5 doc prepackaged level rules and one doc level monitor based on the given rules @@ -1286,13 +1324,163 @@ public void testCreateDetectorForSigmaRuleWithNullCondition() throws IOException List monitorIds = ((List) (detectorMap).get("monitor_id")); assertEquals(1, monitorIds.size()); - indexDoc(index, "1", randomDocWithNullField()); - indexDoc(index, "2", randomDoc()); + indexDoc(index, "1", randomCloudtrailDoc("A12345", "CREATED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + indexDoc(index, "2", randomCloudtrailDoc("A12345", "DELETED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - assertEquals(1, noOfSigmaRuleMatches); + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithDotFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"api.operation\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.event_source\": {\n" + + " \"path\": \"api.service.name\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.aws_region\": {\n" + + " \"path\": \"cloud.region\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithEcsFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); Map params = new HashMap<>(); params.put("detector_id", detectorId); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 43db549c8..c9306d53a 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -80,8 +80,8 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery); - Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_cnt\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } public void testSumAggregationWithGroupBy() throws IOException, SigmaError { @@ -113,7 +113,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { // inputs.query.aggregations -> Query - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); // triggers.bucket_level_trigger.condition -> Condition Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -145,7 +145,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -176,7 +176,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -207,7 +207,83 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } + + public void testCloudtrailAggregationRuleWithDotFields() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"CreateFunction\")) OR ((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"Invoke\"))", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"cloud.region\"},\"aggs\":{\"api_operation\":{\"value_count\":{\"field\":\"api.operation\"}}}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"api_operation\":\"api.operation\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.api_operation > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } } \ No newline at end of file