[Backport 2.15] Backports #1173 and #1178 to 2.x

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java

@@ -3,7 +3,6 @@
 import org.opensearch.commons.alerting.model.Monitor;
 import org.opensearch.commons.alerting.model.MonitorMetadata;
 import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput;
-import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger;
 
 import java.util.List;
 import java.util.Map;
@@ -16,14 +15,18 @@ public class IocScanContext<Data> {
     private final ThreatIntelInput threatIntelInput; // deserialize threat intel input
     private final List<String> indices; // user's log data indices
     private final Map<String, List<String>> iocTypeToIndices;
-    public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices) {
+    private final Map<String, List<String>> concreteIndexToMonitorInputIndicesMap;
+
+    public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices, Map<String,
+            List<String>> concreteIndexToMonitorInputIndicesMap) {
         this.monitor = monitor;
         this.monitorMetadata = monitorMetadata;
         this.dryRun = dryRun;
         this.data = data;
         this.threatIntelInput = threatIntelInput;
         this.indices = indices;
         this.iocTypeToIndices = iocTypeToIndices;
+        this.concreteIndexToMonitorInputIndicesMap = concreteIndexToMonitorInputIndicesMap;
     }
 
     public Monitor getMonitor() {
@@ -50,6 +53,10 @@ public List<String> getIndices() {
         return indices;
     }
 
+    public Map<String, List<String>> getConcreteIndexToMonitorInputIndicesMap() {
+        return concreteIndexToMonitorInputIndicesMap;
+    }
+
     public Map<String, List<String>> getIocTypeToIndices() {
         return iocTypeToIndices;
     }

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java

@@ -39,7 +39,7 @@ public void scanIoCs(IocScanContext<Data> iocScanContext,
             Monitor monitor = iocScanContext.getMonitor();
 
             long startTime = System.currentTimeMillis();
-            IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext.getThreatIntelInput().getPerIocTypeScanInputList());
+            IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext);
             BiConsumer<List<STIX2IOC>, Exception> iocScanResultConsumer = (List<STIX2IOC> maliciousIocs, Exception e) -> {
                 long scanEndTime = System.currentTimeMillis();
                 long timeTaken = scanEndTime - startTime;
@@ -113,31 +113,37 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
      * 4. doc id to iocs map (reverse mapping of 2)
      */
     private IocLookupDtos extractIocsPerType
-    (List<Data> data, List<PerIocTypeScanInput> iocTypeToIndexFieldMappings) {
+    (List<Data> data, IocScanContext<Data> context) {
         Map<String, Set<String>> iocsPerIocTypeMap = new HashMap<>();
         Map<String, Set<String>> iocValueToDocIdMap = new HashMap<>();
         Map<String, Set<String>> docIdToIocsMap = new HashMap<>();
         for (Data datum : data) {
-            for (PerIocTypeScanInput iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) {
+            for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
                 String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
-                String index = getIndexName(datum);
-                List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
-                for (String field : fields) {
-                    List<String> vals = getValuesAsStringList(datum, field);
-                    String id = getId(datum);
-                    String docId = id + ":" + index;
-                    Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
-                    iocs.addAll(vals);
-                    docIdToIocsMap.put(docId, iocs);
-                    for (String ioc : vals) {
-                        Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
-                        docIds.add(docId);
-                        iocValueToDocIdMap.put(ioc, docIds);
-                    }
-                    if (false == vals.isEmpty()) {
-                        iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
+                String concreteIndex = getIndexName(datum);
+                if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)
+                        && false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty()
+                ) {
+                    // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
+                    String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0);
+                    List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
+                    for (String field : fields) {
+                        List<String> vals = getValuesAsStringList(datum, field);
+                        String id = getId(datum);
+                        String docId = id + ":" + index;
+                        Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
                         iocs.addAll(vals);
-                        iocsPerIocTypeMap.put(iocType, iocs);
+                        docIdToIocsMap.put(docId, iocs);
+                        for (String ioc : vals) {
+                            Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
+                            docIds.add(docId);
+                            iocValueToDocIdMap.put(ioc, docIds);
+                        }
+                        if (false == vals.isEmpty()) {
+                            iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
+                            iocs.addAll(vals);
+                            iocsPerIocTypeMap.put(iocType, iocs);
+                        }
                     }
                 }
             }

src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java

@@ -8,6 +8,7 @@
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.settings.Settings;
@@ -34,6 +35,7 @@
 import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService;
 import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner;
 import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
+import org.opensearch.securityanalytics.util.IndexUtils;
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.TransportService;
 
@@ -48,6 +50,7 @@
 import java.util.function.BiConsumer;
 
 import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference;
+import static org.opensearch.securityanalytics.util.IndexUtils.getConcreteindexToMonitorInputIndicesMap;
 
 public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction<DocLevelMonitorFanOutRequest, DocLevelMonitorFanOutResponse> {
     private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class);
@@ -60,6 +63,7 @@ public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAct
 
     private final NamedXContentRegistry xContentRegistry;
     private final SaIoCScanService saIoCScanService;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
     public TransportThreatIntelMonitorFanOutAction(
@@ -70,7 +74,8 @@ public TransportThreatIntelMonitorFanOutAction(
             Settings settings,
             ActionFilters actionFilters,
             SATIFSourceConfigService saTifSourceConfigService,
-            SaIoCScanService saIoCScanService
+            SaIoCScanService saIoCScanService,
+            IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new);
         this.clusterService = clusterService;
@@ -79,6 +84,7 @@ public TransportThreatIntelMonitorFanOutAction(
         this.settings = settings;
         this.saTifSourceConfigService = saTifSourceConfigService;
         this.saIoCScanService = saIoCScanService;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
     }
 
     @Override
@@ -173,14 +179,19 @@ private void onGetIocTypeToIndices(Map<String, List<String>> iocTypeToIndicesMap
                             actionListener.onFailure(e);
                         }
                     };
+                    Map<String, List<String>> concreteindexToMonitorInputIndicesMap = getConcreteindexToMonitorInputIndicesMap(
+                            remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(),
+                            clusterService,
+                            indexNameExpressionResolver);
                     saIoCScanService.scanIoCs(new IocScanContext<>(
                             request.getMonitor(),
                             request.getMonitorMetadata(),
                             false,
                             hits,
                             threatIntelInput,
                             indices,
-                            iocTypeToIndicesMap
+                            iocTypeToIndicesMap,
+                            concreteindexToMonitorInputIndicesMap
                     ), resultConsumer);
                 },
                 e -> {

src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java

@@ -532,7 +532,7 @@ public void getIocTypeToIndices(ActionListener<Map<String, List<String>>> listen
         String stateFieldName = getStateFieldName();
         BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery()
                 .should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString()));
-        stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING));
+        stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING.toString()));
         queryBuilder.must(stateQueryBuilder);
 
         searchRequest.source().query(queryBuilder);

src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java

@@ -4,11 +4,6 @@
  */
 package org.opensearch.securityanalytics.util;
 
-import java.util.Optional;
-import java.util.SortedMap;
-
-import org.opensearch.cluster.metadata.Metadata;
-import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedResponse;
@@ -17,17 +12,22 @@
 import org.opensearch.cluster.metadata.IndexAbstraction;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Predicate;
+import java.util.Optional;
+import java.util.SortedMap;
 
 public class IndexUtils {
 
@@ -210,4 +210,25 @@ public static String getIndexNameWithAlias(ClusterState clusterState, String ali
         return entry.map(Map.Entry::getKey).orElse(null);
     }
 
+    public static Map<String, List<String>> getConcreteindexToMonitorInputIndicesMap(List<String> indices, ClusterService clusterService, IndexNameExpressionResolver resolver) {
+        Map<String, List<String>> result = new HashMap<>();
+
+        for (String index : indices) {
+            String[] concreteIndices = resolver.concreteIndexNames(
+                    clusterService.state(),
+                    IndicesOptions.lenientExpand(),
+                    true,
+                    index
+            );
+            for (String concreteIndex : concreteIndices) {
+                if (!result.containsKey(concreteIndex)) {
+                    result.put(concreteIndex, new ArrayList<>());
+                }
+                result.get(concreteIndex).add(index);
+            }
+        }
+
+        return result;
+    }
+
 }

src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java

@@ -34,8 +34,8 @@
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.commons.ConfigConstants;
-import org.opensearch.commons.alerting.model.action.Action;
 import org.opensearch.commons.alerting.model.ScheduledJob;
+import org.opensearch.commons.alerting.model.action.Action;
 import org.opensearch.commons.alerting.util.IndexUtilsKt;
 import org.opensearch.commons.rest.SecureRestClientBuilder;
 import org.opensearch.core.common.Strings;
@@ -65,8 +65,8 @@
 import org.opensearch.securityanalytics.model.CustomLogType;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.DetectorInput;
-import org.opensearch.securityanalytics.model.DetectorTrigger;
 import org.opensearch.securityanalytics.model.DetectorRule;
+import org.opensearch.securityanalytics.model.DetectorTrigger;
 import org.opensearch.securityanalytics.model.Rule;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
 import org.opensearch.securityanalytics.model.threatintel.IocFinding;
@@ -76,6 +76,7 @@
 import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto;
 import org.opensearch.securityanalytics.util.CorrelationIndices;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
+
 import javax.management.MBeanServerInvocationHandler;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
@@ -97,19 +98,20 @@
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
+
 import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS;
 import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI;
 import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule;
 import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
 import static org.opensearch.securityanalytics.TestHelpers.adLdapLogMappings;
 import static org.opensearch.securityanalytics.TestHelpers.appLogMappings;
 import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule;
-import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
+import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
 import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers;
 import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggersAndType;
-import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
-import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
+import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
 import static org.opensearch.securityanalytics.TestHelpers.s3AccessLogMappings;
+import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
 import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings;
 import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
 import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE;
@@ -756,7 +758,7 @@ private String toJsonString(IocFinding iocFinding) throws IOException {
         return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS)));
     }
 
-   public String toJsonString(ThreatIntelAlert alert) throws IOException {
+    public String toJsonString(ThreatIntelAlert alert) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         return IndexUtilsKt.string(shuffleXContent(alert.toXContent(builder, ToXContent.EMPTY_PARAMS)));
     }
@@ -2009,7 +2011,7 @@ protected String createVpcFlowDetector(String indexName) throws IOException {
         List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
         SearchHit hit = hits.get(0);
 
-        return  ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
+        return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
     }
 
     @SuppressWarnings("unchecked")
@@ -2069,7 +2071,7 @@ protected String createAdLdapDetector(String indexName) throws IOException {
         List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
         SearchHit hit = hits.get(0);
 
-        return  ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
+        return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
     }
 
     @SuppressWarnings("unchecked")
@@ -2228,6 +2230,57 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti
         }
     }
 
+    protected Map<String, Map<String, Boolean>> createTestAlias(
+            String alias, int numOfAliasIndices, boolean includeWriteIndex
+    ) throws IOException {
+        return createTestAlias(
+                alias,
+                randomAliasIndices(alias, numOfAliasIndices, includeWriteIndex),
+                true
+        );
+    }
+
+    protected Map<String, Map<String, Boolean>> createTestAlias(
+            String alias, Map<String, Boolean> indices, boolean createIndices) throws IOException {
+        Map<String, Boolean> indicesMap = new java.util.HashMap<>(indices);
+        Map<String, Map<String, Boolean>> result = new java.util.HashMap<>();
+        XContentBuilder indicesJson = XContentFactory.jsonBuilder()
+                .startObject()
+                .startArray("actions");
+        for (Map.Entry<String, Boolean> entry : indicesMap.entrySet()) {
+            if (createIndices)
+                createTestIndex(entry.getKey(), windowsIndexMapping());
+            boolean isWriteIndex = entry.getValue();
+            indicesJson.startObject()
+                    .startObject("add")
+                    .field("index", entry.getKey())
+                    .field("alias", alias)
+                    .field("is_write_index", isWriteIndex)
+                    .endObject()
+                    .endObject();
+        }
+        indicesJson.endArray().endObject();
+        makeRequest(client(), "POST", "/_aliases", Collections.emptyMap(), new StringEntity(indicesJson.toString(), ContentType.APPLICATION_JSON));
+        result.put(alias, indicesMap);
+        return result;
+    }
+
+
+    protected static Map<String, Boolean> randomAliasIndices(
+            String alias, int num, boolean includeWriteIndex) {
+        Map<String, Boolean> indices = new HashMap<>();
+        int writeIndex = randomIntBetween(0, num - 1);
+        for (int i = 0; i < num; i++) {
+            String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+            while (indexName.equals(alias) || indices.containsKey(indexName)) {
+                indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+            }
+            boolean isWriteIndex = includeWriteIndex && i == writeIndex;
+            indices.put(indexName, isWriteIndex);
+        }
+        return indices;
+    }
+
     public static class LogIndices {
         public String vpcFlowsIndex;
         public String adLdapLogsIndex;