[Backport 2.11] Backport #789, #788, #867, and #918 to 2.11

src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java

@@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) {
         "\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+
         "\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+
         "\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+
-        "\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+
-        "\u0000\t\n\f\r  n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+
-        "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+
-        "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+
-        "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+
-        "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+
-        "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+
-        "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+
-        "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+
-        "\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+
-        "&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+
-        "\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+
-        "\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+
-        "\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+
-        "\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+
-        "\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+
-        ">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+
-        "=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+
-        "\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+
-        "\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+
-        "\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+
-        "u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+
-        "\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+
-        "\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+
-        "\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+
-        "\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+
-        "\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+
-        "C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+
-        "F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+
-        "\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+
-        "-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+
-        "\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+
-        "PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+
-        "\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+
-        "\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+
-        "\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+
-        "\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+
-        "\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+
-        "\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+
-        "\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+
-        "\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+
-        "\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+
-        "\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+
-        "\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
+        "\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+
+        "z\u0003\u0000\t\n\f\r  n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+
+        "\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+
+        "\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+
+        "\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+
+        "\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+
+        "\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+
+        "\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+
+        "\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+
+        "\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+
+        "\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+
+        "\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+
+        "\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+
+        "@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+
+        "\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+
+        "\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+
+        "\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+
+        "$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+
+        "\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+
+        "*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+
+        "-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+
+        "01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+
+        "\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+
+        "\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+
+        "i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+
+        "m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+
+        "\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+
+        "\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+
+        "\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+
+        "\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+
+        "KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+
+        "MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+
+        "\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+
+        "\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+
+        "\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+
+        "\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+
+        "YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+
+        "\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+
+        "\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+
+        "_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+
+        "\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
+        "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
+        "fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+
+        "\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
     public static final ATN _ATN =
         new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {

src/main/grammars/Aggregation.g4

@@ -21,7 +21,7 @@ RPAREN : ')' ;
 
 DECIMAL : '-'?[0-9]+('.'[0-9]+)? ;
 
-IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ;
+IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ;
 WS : [ \r\t\u000C\n]+ -> skip ;
 
 comparison_expr : comparison_operand comp_operator comparison_operand   # ComparisonExpressionWithOperator

src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java

@@ -348,15 +348,26 @@
         List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
         fieldMappingDocs.forEach( newFieldMapping -> {
             Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
-            for (FieldMappingDoc e: existingFieldMappings) {
-                if (e.getRawField().equals(newFieldMapping.getRawField())) {
+            for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
+                if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
                     if ((
-                            e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
-                                    e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
+                            existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
+                                    existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
                     ) || (
-                            e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
+                            existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
                     )) {
-                        foundFieldMappingDoc = Optional.of(e);
+                        foundFieldMappingDoc = Optional.of(existingFieldMapping);
+                    }
+                    // Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
+                    // Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
+                } else {
+                    String id = existingFieldMapping.getId();
+                    int indexOfPipe = id.indexOf("|");
+                    if (indexOfPipe != -1) {
+                        String ecsIdField = id.substring(indexOfPipe + 1);
+                        if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
+                            foundFieldMappingDoc = Optional.of(existingFieldMapping);
+                        }
                     }
                 }
             }

src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java

@@ -18,7 +18,6 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.OpenSearchStatusException;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.admin.indices.get.GetIndexRequest;
 import org.opensearch.action.admin.indices.get.GetIndexResponse;
@@ -477,13 +476,16 @@
                             String rawPath = requiredField.getRawField();
                             String ocsfPath = requiredField.getOcsf();
                             if (allFieldsFromIndex.contains(rawPath)) {
-                                if (alias != null) {
-                                    // Maintain list of found paths in index
-                                    applyableAliases.add(alias);
-                                } else {
-                                    applyableAliases.add(rawPath);
+                                // if the alias was already added into applyable aliases, then skip to avoid duplicates
+                                if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) {
+                                    if (alias != null) {
+                                        // Maintain list of found paths in index
+                                        applyableAliases.add(alias);
+                                    } else {
+                                        applyableAliases.add(rawPath);
+                                    }
+                                    pathsOfApplyableAliases.add(rawPath);
                                 }
-                                pathsOfApplyableAliases.add(rawPath);
                             } else if (allFieldsFromIndex.contains(ocsfPath)) {
                                 applyableAliases.add(alias);
                                 pathsOfApplyableAliases.add(ocsfPath);
@@ -497,13 +499,21 @@
                             }
                         }
 
+                        // turn unmappedFieldAliases into a set to remove duplicates
+                        Set<String> setOfUnmappedFieldAliases = new HashSet<>(unmappedFieldAliases);
+
+                        // filter out aliases that were included in applyableAliases already
+                        List<String> filteredUnmappedFieldAliases = setOfUnmappedFieldAliases.stream()
+                                .filter(e -> false == applyableAliases.contains(e))
+                                .collect(Collectors.toList());
+
                         Map<String, Map<String, String>> aliasMappingFields = new HashMap<>();
                         XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject();
                         for (LogType.Mapping mapping: requiredFields) {
                             if (allFieldsFromIndex.contains(mapping.getOcsf())) {
                                 aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
                             } else if (mapping.getEcs() != null) {
-                                aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
+                                shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
                             } else if (mapping.getEcs() == null) {
                                 aliasMappingFields.put(mapping.getRawField(), Map.of("type", "alias", "path", mapping.getRawField()));
                             }
@@ -520,7 +530,7 @@
                                 .collect(Collectors.toList());
 
                         actionListener.onResponse(
-                                new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases)
+                                new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases)
                         );
                     } catch (Exception e) {
                         actionListener.onFailure(e);
@@ -534,6 +544,26 @@
         });
     }
 
+    /**
+     * Only updates the alias mapping fields if the ecs key has not been mapped yet
+     * or if pathOfApplyableAliases contains the raw field
+     *
+     * @param mapping
+     * @param aliasMappingFields
+     * @param pathsOfApplyableAliases
+     */
+    private static void shouldUpdateEcsMappingAndMaybeUpdates(LogType.Mapping mapping, Map<String, Map<String, String>> aliasMappingFields, List<String> pathsOfApplyableAliases) {
+        // check if aliasMappingFields already contains a key
+        if (aliasMappingFields.containsKey(mapping.getEcs())) {
+            // if the pathOfApplyableAliases contains the raw field, then override the existing map
+            if (pathsOfApplyableAliases.contains(mapping.getRawField())) {
+                aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
+            }
+        } else {
+            aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
+        }
+    }
+
     /**
      * Given index name, resolves it to single concrete index, depending on what initial <code>indexName</code> is.
      * In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned.

src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java

@@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat
                 aggregationBuilder = new TermsAggregationBuilder(name).field(name);
                 break;
             case "count":
-                aggregationBuilder = new ValueCountAggregationBuilder(name).field(name);
+                aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name);
                 break;
             default:
                 throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction));