From 1483883ca996d72dd5c767081c22c1fd4a42bdf9 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 8 Aug 2024 15:02:11 -0700 Subject: [PATCH] Bug fixes for threat intel (#1223) * fix default source config behavior and release lock Signed-off-by: Joanne Wang * add check for multinode index not exist Signed-off-by: Joanne Wang * add tests Signed-off-by: Joanne Wang * replace match query with match phrase query in threat intel to avoid tokenization/analysis of value Signed-off-by: Surya Sashank Nistala * added test to verify deactiavte ioc_upload Signed-off-by: Surya Sashank Nistala * add debug log to release lock Signed-off-by: Joanne Wang * fix test Signed-off-by: Joanne Wang * fix update url download and add test Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang Signed-off-by: Surya Sashank Nistala Co-authored-by: Surya Sashank Nistala --- .../threatIntel/common/TIFLockService.java | 1 + .../DefaultTifSourceConfigLoaderService.java | 13 +- .../SATIFSourceConfigManagementService.java | 46 ++- .../service/SATIFSourceConfigService.java | 6 - .../TransportGetIocFindingsAction.java | 3 +- .../TransportIndexTIFSourceConfigAction.java | 5 - .../TransportGetThreatIntelAlertsAction.java | 6 +- ...ortUpdateThreatIntelAlertStatusAction.java | 16 +- .../SourceConfigWithoutS3RestApiIT.java | 283 ++++++++++++++++++ .../ThreatIntelMonitorRestApiIT.java | 8 + .../common/ThreatIntelLockServiceTests.java | 9 +- 11 files changed, 369 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index 98abf040a..8b3917791 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -74,6 +74,7 @@ public void onFailure(final Exception e) { * @param lockModel the lock model */ public void releaseLock(final LockModel lockModel) { + log.debug("Releasing lock with id [{}]", lockModel.getLockId()); lockService.release( lockModel, ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java index ed160bbf1..c247109d6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -6,6 +6,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.client.Client; import org.opensearch.core.action.ActionListener; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -21,6 +22,7 @@ import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.transport.RemoteTransportException; import java.net.URL; import java.time.Instant; @@ -31,8 +33,11 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + //todo handle refresh, update tif config // todo block creation of url based config in transport layer + public class DefaultTifSourceConfigLoaderService { private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); private final BuiltInTIFMetadataLoader tifMetadataLoader; @@ -64,8 +69,12 @@ public void createDefaultTifConfigsIfNotExists(ActionListener listener) { ActionListener.wrap(searchResponse -> { createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); }, e -> { - log.error("Failed to search tif config index for default tif configs", e); - listener.onFailure(e); + if (e instanceof IndexNotFoundException || (e instanceof RemoteTransportException && e.getCause() instanceof IndexNotFoundException)) { + createTifConfigsThatDontExist(getEmptySearchResponse(), tifMetadataList, listener); + } else { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + } })); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index de483deea..2e2c0a5c6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -32,6 +32,7 @@ import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; @@ -39,6 +40,7 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.time.Instant; @@ -275,6 +277,48 @@ public void updateIocAndTIFSourceConfig( try { saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap( retrievedSaTifSourceConfig -> { + // Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request. + // If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API. + if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) { + SATIFSourceConfig config = new SATIFSourceConfig( + retrievedSaTifSourceConfig.getId(), + retrievedSaTifSourceConfig.getVersion(), + retrievedSaTifSourceConfig.getName(), + retrievedSaTifSourceConfig.getFormat(), + retrievedSaTifSourceConfig.getType(), + retrievedSaTifSourceConfig.getDescription(), + retrievedSaTifSourceConfig.getCreatedByUser(), + retrievedSaTifSourceConfig.getCreatedAt(), + retrievedSaTifSourceConfig.getSource(), + retrievedSaTifSourceConfig.getEnabledTime(), + retrievedSaTifSourceConfig.getLastUpdateTime(), + retrievedSaTifSourceConfig.getSchedule(), + retrievedSaTifSourceConfig.getState(), + retrievedSaTifSourceConfig.getRefreshType(), + Instant.now(), + updatedByUser, + retrievedSaTifSourceConfig.isEnabled(), + retrievedSaTifSourceConfig.getIocStoreConfig(), + retrievedSaTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan + ); + internalUpdateTIFSourceConfig(config, ActionListener.wrap( + r -> { + listener.onResponse(new SATIFSourceConfigDto(r)); + }, e -> { + String action = saTifSourceConfigDto.isEnabledForScan() ? "activate" : "deactivate"; + log.error(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e))); + return; + } + )); + return; + } else if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource) { // fail if enabled_for_scan isn't changed and type is url download + log.error("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType()); + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + return; + } + if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) { log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()); listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( @@ -458,7 +502,7 @@ public void deleteTIFSourceConfig( }, e -> { log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId); if (e instanceof IndexNotFoundException) { - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); } else { listener.onFailure(e); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 34be1e58e..ed58e3371 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -255,12 +255,6 @@ public void searchTIFSourceConfigs( ) { SearchRequest searchRequest = getSearchRequest(searchSourceBuilder); - // Check to make sure the job index exists - if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST))); - return; - } - client.search(searchRequest, ActionListener.wrap( searchResponse -> { if (searchResponse.isTimedOut()) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java index f27472eeb..4e6d2f349 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -120,7 +120,8 @@ protected void doExecute(Task task, GetIocFindingsRequest request, ActionListene List iocIds = request.getIocIds(); if (iocIds != null && !iocIds.isEmpty()) { BoolQueryBuilder iocIdQueryBuilder = QueryBuilders.boolQuery(); - iocIds.forEach(it -> iocIdQueryBuilder.should(QueryBuilders.matchQuery("ioc_feed_ids.ioc_id", it))); + // can't use match query because it analyzes the value and considers `hyphens` as word separators + iocIds.forEach(it -> iocIdQueryBuilder.should(QueryBuilders.matchPhraseQuery("ioc_feed_ids.ioc_id", it))); queryBuilder.filter(iocIdQueryBuilder); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index 77ea09a4c..440191146 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -99,11 +99,6 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques } try { SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); - if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource - && request.getMethod().equals(RestRequest.Method.POST)) { - listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); - return; - } saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( saTifSourceConfigDto, lock, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java index ac1afe4f9..9e8e29f62 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java @@ -108,8 +108,8 @@ protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionL SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( searchResponse -> { @@ -141,7 +141,7 @@ private void getAlerts(List monitorIds, BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { monitorIdMatchQuery.should(QueryBuilders.boolQuery() - .must(QueryBuilders.matchQuery("monitor_id", monitorId))); + .must(QueryBuilders.matchPhraseQuery("monitor_id", monitorId))); } queryBuilder.filter(monitorIdMatchQuery); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java index 7902453b9..7aa828d0c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java @@ -102,8 +102,8 @@ protected void doExecute(Task task, UpdateThreatIntelAlertStatusRequest request, SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( searchResponse -> { @@ -174,22 +174,22 @@ private static SearchSourceBuilder getSearchSourceQueryingForAlertsToUpdate(List BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { - monitorIdMatchQuery.should(QueryBuilders.matchQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); } queryBuilder.filter(monitorIdMatchQuery); BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); for (String id : request.getAlertIds()) { - idMatchQuery.should(QueryBuilders.matchQuery("_id", id)); + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); } queryBuilder.filter(idMatchQuery); if (request.getState() == Alert.State.COMPLETED) { - queryBuilder.filter(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); } else if (request.getState() == Alert.State.ACKNOWLEDGED) { - queryBuilder.filter(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); } else { log.error("Threat intel monitor not found. No alerts to update"); listener.onFailure(new SecurityAnalyticsException("Threat intel monitor not found. No alerts to update", @@ -274,14 +274,14 @@ private static SearchSourceBuilder getSearchSourceQueryingForUpdatedAlerts(List< BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { - monitorIdMatchQuery.should(QueryBuilders.matchQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); } queryBuilder.filter(monitorIdMatchQuery); BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); for (String id : alertIds) { - idMatchQuery.should(QueryBuilders.matchQuery("_id", id)); + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); } queryBuilder.filter(idMatchQuery); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index f00c8e9a8..39377262e 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -16,6 +16,7 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; @@ -26,9 +27,13 @@ import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.util.STIX2IOCGenerator; import java.io.IOException; +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -359,6 +364,203 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc Thread.sleep(10000); } + public void testActivateDeactivateIocUploadSourceConfig() throws IOException, InterruptedException { + // Create source config with IPV4 IOCs + String feedName = "test_update"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "1", + "ioc", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // create source config with ipv4 ioc type + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs by feed Ids + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + // update source config to contain only hashes as an ioc type + iocs = Collections.emptyList(); + + iocUploadSource = new IocUploadSource(null, iocs); + iocTypes = List.of("hashes"); + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + Thread.sleep(10000); + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(2, findingIndices.size()); + + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response - there should only be 1 ioc indexed according to the ioc type + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(1, totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(1, iocHits.size()); + Thread.sleep(10000); + } + + public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, InterruptedException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + // update default source config with enabled_for_scan updated + Response response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + + // Ensure that only 1 ioc index is present from default source + List findingIndices = getIocIndices(); + Assert.assertEquals(1, findingIndices.size()); + + Thread.sleep(100); // TODO: pass in action listener when releasing lock + + // try to update default source config again to ensure operation is not accepted when enabled_for_scan is unchanged + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + } + public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException { String feedName = "test_ioc_upload"; String feedFormat = "STIX"; @@ -586,4 +788,85 @@ public void testSearchIocUploadSourceConfig() throws IOException { Assert.assertEquals(2, ((Map) ((Map) respMap.get("hits")).get("total")).get("value")); } + public void testSearchAndCreateDefaultSourceConfig() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + } + + public void testUpdateDefaultSourceConfigThrowsError() throws IOException, InterruptedException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + + Thread.sleep(100); // TODO: pass in action listener when releasing lock + + // update default source config again to ensure lock was released + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 12c4aadaf..d3073e9dd 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -246,6 +246,14 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(2, numFindings); }); + // Use ListIOCs API with large size to ensure matchQuery related bug is not throwing too many bool clauses exception + listIocsUri = String.format("?%s=%s", "size", 1000); + listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI, Collections.emptyMap(), null); + assertEquals(200, listIocsResponse.getStatusLine().getStatusCode()); + listIocsResponseMap = responseAsMap(listIocsResponse); + iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertTrue(2 < iocsMap.size()); // number should be greater than custom source iocs because of default config + //alerts via system index search searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); Assert.assertEquals(4, searchHits.size()); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java index 7a95e746f..ba6f54926 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -52,7 +52,14 @@ public void testAcquireLock_whenCalled_thenNotBlocked() { public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - noOpsLockService.releaseLock(null); + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + noOpsLockService.releaseLock(lockModel); } public void testRenewLock_whenCalled_thenNotBlocked() {