From e88f4e61e640ac5fab13ed66d8bcf689bfbae4f1 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Thu, 18 Jul 2024 13:06:19 -0700 Subject: [PATCH] Backports #1173 and #1178 to 2.x (#1180) * [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs (#1173) * resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs Signed-off-by: Surya Sashank Nistala * clear indices after test Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala * Fix match query in search tif source configs request to use string value of enum (#1178) Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala (cherry picked from commit f7cfae5a38766048079fd1e027a2a2379f0a5b8a) --- .../iocscan/dto/IocScanContext.java | 11 +- .../iocscan/service/IoCScanService.java | 46 +++--- ...ansportThreatIntelMonitorFanOutAction.java | 15 +- .../service/SATIFSourceConfigService.java | 2 +- .../securityanalytics/util/IndexUtils.java | 33 ++++- .../SecurityAnalyticsRestTestCase.java | 69 ++++++++- .../ThreatIntelMonitorRestApiIT.java | 139 +++++++++++++++++- 7 files changed, 272 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java index d04a85bc5..4a062e718 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java @@ -3,7 +3,6 @@ import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.commons.alerting.model.MonitorMetadata; import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; -import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; import java.util.List; import java.util.Map; @@ -16,7 +15,10 @@ public class IocScanContext { private final ThreatIntelInput threatIntelInput; // deserialize threat intel input private final List indices; // user's log data indices private final Map> iocTypeToIndices; - public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List data, ThreatIntelInput threatIntelInput, List indices, Map> iocTypeToIndices) { + private final Map> concreteIndexToMonitorInputIndicesMap; + + public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List data, ThreatIntelInput threatIntelInput, List indices, Map> iocTypeToIndices, Map> concreteIndexToMonitorInputIndicesMap) { this.monitor = monitor; this.monitorMetadata = monitorMetadata; this.dryRun = dryRun; @@ -24,6 +26,7 @@ public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean this.threatIntelInput = threatIntelInput; this.indices = indices; this.iocTypeToIndices = iocTypeToIndices; + this.concreteIndexToMonitorInputIndicesMap = concreteIndexToMonitorInputIndicesMap; } public Monitor getMonitor() { @@ -50,6 +53,10 @@ public List getIndices() { return indices; } + public Map> getConcreteIndexToMonitorInputIndicesMap() { + return concreteIndexToMonitorInputIndicesMap; + } + public Map> getIocTypeToIndices() { return iocTypeToIndices; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index 7578699e0..82d0853a3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -39,7 +39,7 @@ public void scanIoCs(IocScanContext iocScanContext, Monitor monitor = iocScanContext.getMonitor(); long startTime = System.currentTimeMillis(); - IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext.getThreatIntelInput().getPerIocTypeScanInputList()); + IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext); BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { long scanEndTime = System.currentTimeMillis(); long timeTaken = scanEndTime - startTime; @@ -113,31 +113,37 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( * 4. doc id to iocs map (reverse mapping of 2) */ private IocLookupDtos extractIocsPerType - (List data, List iocTypeToIndexFieldMappings) { + (List data, IocScanContext context) { Map> iocsPerIocTypeMap = new HashMap<>(); Map> iocValueToDocIdMap = new HashMap<>(); Map> docIdToIocsMap = new HashMap<>(); for (Data datum : data) { - for (PerIocTypeScanInput iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) { + for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) { String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); - String index = getIndexName(datum); - List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); - for (String field : fields) { - List vals = getValuesAsStringList(datum, field); - String id = getId(datum); - String docId = id + ":" + index; - Set iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>()); - iocs.addAll(vals); - docIdToIocsMap.put(docId, iocs); - for (String ioc : vals) { - Set docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>()); - docIds.add(docId); - iocValueToDocIdMap.put(ioc, docIds); - } - if (false == vals.isEmpty()) { - iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + String concreteIndex = getIndexName(datum); + if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex) + && false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty() + ) { + // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc. + String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0); + List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + for (String field : fields) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>()); iocs.addAll(vals); - iocsPerIocTypeMap.put(iocType, iocs); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } } } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java index 012d4b1de..6864f7a98 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java @@ -8,6 +8,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; @@ -34,6 +35,7 @@ import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService; import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -48,6 +50,7 @@ import java.util.function.BiConsumer; import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference; +import static org.opensearch.securityanalytics.util.IndexUtils.getConcreteindexToMonitorInputIndicesMap; public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction { private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class); @@ -60,6 +63,7 @@ public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAct private final NamedXContentRegistry xContentRegistry; private final SaIoCScanService saIoCScanService; + private final IndexNameExpressionResolver indexNameExpressionResolver; @Inject public TransportThreatIntelMonitorFanOutAction( @@ -70,7 +74,8 @@ public TransportThreatIntelMonitorFanOutAction( Settings settings, ActionFilters actionFilters, SATIFSourceConfigService saTifSourceConfigService, - SaIoCScanService saIoCScanService + SaIoCScanService saIoCScanService, + IndexNameExpressionResolver indexNameExpressionResolver ) { super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new); this.clusterService = clusterService; @@ -79,6 +84,7 @@ public TransportThreatIntelMonitorFanOutAction( this.settings = settings; this.saTifSourceConfigService = saTifSourceConfigService; this.saIoCScanService = saIoCScanService; + this.indexNameExpressionResolver = indexNameExpressionResolver; } @Override @@ -173,6 +179,10 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap actionListener.onFailure(e); } }; + Map> concreteindexToMonitorInputIndicesMap = getConcreteindexToMonitorInputIndicesMap( + remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(), + clusterService, + indexNameExpressionResolver); saIoCScanService.scanIoCs(new IocScanContext<>( request.getMonitor(), request.getMonitorMetadata(), @@ -180,7 +190,8 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap hits, threatIntelInput, indices, - iocTypeToIndicesMap + iocTypeToIndicesMap, + concreteindexToMonitorInputIndicesMap ), resultConsumer); }, e -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index b3f47e303..35571facd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -532,7 +532,7 @@ public void getIocTypeToIndices(ActionListener>> listen String stateFieldName = getStateFieldName(); BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery() .should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString())); - stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING)); + stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING.toString())); queryBuilder.must(stateQueryBuilder); searchRequest.source().query(queryBuilder); diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index a24286fda..9d5066308 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -4,11 +4,6 @@ */ package org.opensearch.securityanalytics.util; -import java.util.Optional; -import java.util.SortedMap; - -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -17,17 +12,22 @@ import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.function.Predicate; +import java.util.Optional; +import java.util.SortedMap; public class IndexUtils { @@ -210,4 +210,25 @@ public static String getIndexNameWithAlias(ClusterState clusterState, String ali return entry.map(Map.Entry::getKey).orElse(null); } + public static Map> getConcreteindexToMonitorInputIndicesMap(List indices, ClusterService clusterService, IndexNameExpressionResolver resolver) { + Map> result = new HashMap<>(); + + for (String index : indices) { + String[] concreteIndices = resolver.concreteIndexNames( + clusterService.state(), + IndicesOptions.lenientExpand(), + true, + index + ); + for (String concreteIndex : concreteIndices) { + if (!result.containsKey(concreteIndex)) { + result.put(concreteIndex, new ArrayList<>()); + } + result.get(concreteIndex).add(index); + } + } + + return result; + } + } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index db80846ca..2847b600a 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -34,8 +34,8 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.commons.ConfigConstants; -import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.model.ScheduledJob; +import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.util.IndexUtilsKt; import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.core.common.Strings; @@ -65,8 +65,8 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.model.threatintel.IocFinding; @@ -76,6 +76,7 @@ import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; + import javax.management.MBeanServerInvocationHandler; import javax.management.MalformedObjectNameException; import javax.management.ObjectName; @@ -97,6 +98,7 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.stream.Collectors; + import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; @@ -104,12 +106,12 @@ import static org.opensearch.securityanalytics.TestHelpers.adLdapLogMappings; import static org.opensearch.securityanalytics.TestHelpers.appLogMappings; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggersAndType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.s3AccessLogMappings; +import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; @@ -756,7 +758,7 @@ private String toJsonString(IocFinding iocFinding) throws IOException { return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); } - public String toJsonString(ThreatIntelAlert alert) throws IOException { + public String toJsonString(ThreatIntelAlert alert) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(alert.toXContent(builder, ToXContent.EMPTY_PARAMS))); } @@ -2009,7 +2011,7 @@ protected String createVpcFlowDetector(String indexName) throws IOException { List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); } @SuppressWarnings("unchecked") @@ -2069,7 +2071,7 @@ protected String createAdLdapDetector(String indexName) throws IOException { List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); } @SuppressWarnings("unchecked") @@ -2228,6 +2230,57 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti } } + protected Map> createTestAlias( + String alias, int numOfAliasIndices, boolean includeWriteIndex + ) throws IOException { + return createTestAlias( + alias, + randomAliasIndices(alias, numOfAliasIndices, includeWriteIndex), + true + ); + } + + protected Map> createTestAlias( + String alias, Map indices, boolean createIndices) throws IOException { + Map indicesMap = new java.util.HashMap<>(indices); + Map> result = new java.util.HashMap<>(); + XContentBuilder indicesJson = XContentFactory.jsonBuilder() + .startObject() + .startArray("actions"); + for (Map.Entry entry : indicesMap.entrySet()) { + if (createIndices) + createTestIndex(entry.getKey(), windowsIndexMapping()); + boolean isWriteIndex = entry.getValue(); + indicesJson.startObject() + .startObject("add") + .field("index", entry.getKey()) + .field("alias", alias) + .field("is_write_index", isWriteIndex) + .endObject() + .endObject(); + } + indicesJson.endArray().endObject(); + makeRequest(client(), "POST", "/_aliases", Collections.emptyMap(), new StringEntity(indicesJson.toString(), ContentType.APPLICATION_JSON)); + result.put(alias, indicesMap); + return result; + } + + + protected static Map randomAliasIndices( + String alias, int num, boolean includeWriteIndex) { + Map indices = new HashMap<>(); + int writeIndex = randomIntBetween(0, num - 1); + for (int i = 0; i < num; i++) { + String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + while (indexName.equals(alias) || indices.containsKey(indexName)) { + indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + } + boolean isWriteIndex = includeWriteIndex && i == writeIndex; + indices.put(indexName, isWriteIndex); + } + return indices; + } + public static class LogIndices { public String vpcFlowsIndex; public String adLdapLogsIndex; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 5ffd37cef..f81b913e7 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -2,8 +2,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.client.Response; import org.opensearch.common.settings.Settings; @@ -156,14 +154,15 @@ private void indexTifSourceConfig(int num, String configId, String indexPattern, Response response = indexDoc(indexName, configId, config.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); } - public void testCreateThreatIntelMonitor() throws IOException { + public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); indexSourceConfigsAndIocs(1, vals); - String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String index = "alias1"; + Map> testAlias = createTestAlias(index, 1, true); String monitorName = "test_monitor_name"; @@ -283,8 +282,135 @@ public void testCreateThreatIntelMonitor() throws IOException { totalHits = (HashMap) hits.get("total"); totalHitsVal = (Integer) totalHits.get("value"); assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitor() throws IOException { + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + indexSourceConfigsAndIocs(1, vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + assertEquals(1, 1); + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); } public static String getMatchAllRequest() { @@ -311,5 +437,10 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { null, List.of(t1, t2, t3, t4)); } + + @Override + protected boolean preserveIndicesUponCompletion() { + return false; + } }