diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 36fa71a2a..9349bc020 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -497,7 +497,7 @@ public List> getSettings() { SecurityAnalyticsSettings.BATCH_SIZE, SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, - SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS + SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN ); } diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java index 9473dbbfc..f068be77c 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java @@ -21,17 +21,11 @@ import org.opensearch.securityanalytics.action.ListIOCsAction; import org.opensearch.securityanalytics.action.ListIOCsActionRequest; import org.opensearch.securityanalytics.action.ListIOCsActionResponse; -import org.opensearch.securityanalytics.commons.model.STIX2; -import org.opensearch.securityanalytics.model.STIX2IOC; import java.io.IOException; -import java.time.DateTimeException; -import java.time.Instant; import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; - public class RestListIOCsAction extends BaseRestHandler { private static final Logger log = LogManager.getLogger(RestListIOCsAction.class); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 2b3b108df..8c5769309 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -10,18 +10,14 @@ import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.admin.indices.rollover.RolloverRequest; -import org.opensearch.action.admin.indices.rollover.RolloverResponse; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.Streams; @@ -29,6 +25,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.securityanalytics.commons.model.IOC; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.UpdateAction; import org.opensearch.securityanalytics.commons.store.FeedStore; import org.opensearch.securityanalytics.model.STIX2IOC; @@ -36,7 +33,6 @@ import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; -import org.opensearch.securityanalytics.util.IndexUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -56,7 +52,7 @@ public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID"; public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER; - public static final String IOC_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE; + public static final String IOC_ALL_INDEX_PATTERN_BY_ID = IOC_INDEX_NAME_TEMPLATE + "-*"; public static final String IOC_TIME_PLACEHOLDER = "TIME"; public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER; @@ -117,80 +113,36 @@ public void storeIOCs(Map actionToIOCs) { } public void indexIocs(List iocs) throws IOException { - String iocAlias = getIocIndexAlias(saTifSourceConfig.getId()); - String iocPattern = getIocIndexRolloverPattern(saTifSourceConfig.getId()); + String newActiveIndex = getNewActiveIndex(saTifSourceConfig.getId()); + String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId()); - if (iocIndexExists(iocAlias) == false) { - initFeedIndex(iocAlias, iocPattern, ActionListener.wrap( - r -> { - saTifSourceConfig.getIocTypes().forEach(type -> { - String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state()); - String lowerCaseType = type.toLowerCase(Locale.ROOT); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().putIfAbsent(lowerCaseType, new ArrayList<>()); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(iocAlias); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex); - }); - bulkIndexIocs(iocs, iocAlias); - }, e-> { - log.error("Failed to initialize the IOC index and save the IOCs", e); - baseListener.onFailure(e); - } - )); - } else { - rolloverIndex(iocAlias, iocPattern, ActionListener.wrap( - r -> { - saTifSourceConfig.getIocTypes().forEach(type -> { - String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state()); - String lowerCaseType = type.toLowerCase(Locale.ROOT); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex); - }); - bulkIndexIocs(iocs, iocAlias); - }, e -> { - log.error("Failed to rollover the IOC index and save the IOCs", e); - baseListener.onFailure(e); - } - )); - } - } - - private void rolloverIndex( - String alias, - String pattern, - ActionListener listener - ) { - if (clusterService.state().metadata().hasAlias(alias) == false) { - listener.onFailure(new OpenSearchException("Alias not initialized")); - return; - } - - RolloverRequest request = new RolloverRequest(alias, pattern); - request.getCreateIndexRequest() - .mapping(iocIndexMapping()) - .settings(Settings.builder().put("index.hidden", true).build()); - client.admin().indices().rolloverIndex( - request, - ActionListener.wrap( - rolloverResponse -> { - if (false == rolloverResponse.isRolledOver()) { - log.info(alias + "not rolled over. Rollover condition status: " + rolloverResponse.getConditionStatus()); - listener.onFailure(new OpenSearchException(alias + "not rolled over. Rollover condition status: " + rolloverResponse.getConditionStatus())); - } else { - listener.onResponse(rolloverResponse); - } - }, e -> { - log.error("rollover failed for alias [" + alias + "]."); - listener.onFailure(e); + initFeedIndex(newActiveIndex, ActionListener.wrap( + r -> { + saTifSourceConfig.getIocTypes().forEach(type -> { + IOCType iocType = IOCType.fromString(type); + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + List listOfIocToIndexDetails = + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails(); + listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType); + DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails = + new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex); + listOfIocToIndexDetails.add(iocToIndexDetails); } - ) - ); + }); + bulkIndexIocs(iocs, newActiveIndex); + }, e-> { + log.error("Failed to initialize the IOC index and save the IOCs", e); + baseListener.onFailure(e); + } + )); } - private void bulkIndexIocs(List iocs, String iocAlias) throws IOException { + private void bulkIndexIocs(List iocs, String activeIndex) throws IOException { List bulkRequestList = new ArrayList<>(); BulkRequest bulkRequest = new BulkRequest(); for (STIX2IOC ioc : iocs) { - IndexRequest indexRequest = new IndexRequest(iocAlias) + IndexRequest indexRequest = new IndexRequest(activeIndex) .id(StringUtils.isBlank(ioc.getId())? UUID.randomUUID().toString() : ioc.getId()) .opType(DocWriteRequest.OpType.INDEX) .source(ioc.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); @@ -235,27 +187,20 @@ private void bulkIndexIocs(List iocs, String iocAlias) throws IOExcept } } - public boolean iocIndexExists(String alias) { - ClusterState clusterState = clusterService.state(); - return clusterState.metadata().hasAlias(alias); + public static String getAllIocIndexPatternById(String sourceConfigId) { + return IOC_ALL_INDEX_PATTERN_BY_ID.replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT)); } - public static String getIocIndexAlias(String feedSourceConfigId) { - return IOC_WRITE_INDEX_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); - } - - public static String getIocIndexRolloverPattern(String feedSourceConfigId) { + public static String getNewActiveIndex(String sourceConfigId) { return IOC_INDEX_PATTERN - .replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)) + .replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT)) .replace(IOC_TIME_PLACEHOLDER, Long.toString(Instant.now().toEpochMilli())); } - - public void initFeedIndex(String feedAliasName, String feedIndexName, ActionListener listener) { + public void initFeedIndex(String feedIndexName, ActionListener listener) { var indexRequest = new CreateIndexRequest(feedIndexName) .mapping(iocIndexMapping()) .settings(Settings.builder().put("index.hidden", true).build()); - indexRequest.alias(new Alias(feedAliasName)); // set the alias client.admin().indices().create(indexRequest, ActionListener.wrap( r -> { log.info("Created system index {}", feedIndexName); diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 83bc8e567..b0f0fed74 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -230,7 +230,7 @@ public static final List> settings() { Setting.Property.NodeScope, Setting.Property.Dynamic ); - public static final Setting IOC_MAX_INDICES_PER_ALIAS = Setting.intSetting( + public static final Setting IOC_MAX_INDICES_PER_INDEX_PATTERN = Setting.intSetting( "plugins.security_analytics.ioc.max_indices_per_alias", 30, 1, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index 6bcd483fe..c08b74eea 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -5,23 +5,36 @@ package org.opensearch.securityanalytics.threatIntel.common; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; /** * Source config dto validator */ public class SourceConfigDtoValidator { public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) { - List errorMsgs = new ArrayList<>(); + List iocTypeEnumNames = Arrays.stream(IOCType.values()) + .map(Enum::name) + .collect(Collectors.toList()); + if (sourceConfigDto.getIocTypes().isEmpty()) { errorMsgs.add("Must specify at least one IOC type"); + } else { + for (String s: sourceConfigDto.getIocTypes()) { + if (false == iocTypeEnumNames.contains(s)) { + errorMsgs.add("Invalid IOC type: " + s); + } + } } + switch (sourceConfigDto.getType()) { case IOC_UPLOAD: if (sourceConfigDto.isEnabled()) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java index 8e60e106d..60c749ca4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java @@ -9,78 +9,72 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.commons.model.IOCType; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; /** * Model used for the default IOC store configuration - * Stores the IOC mapping in a map of string to list of strings + * Stores the IOC mapping in a list of IocToIndexDetails which contains the ioc type, index pattern, and the active index */ public class DefaultIocStoreConfig extends IocStoreConfig implements Writeable, ToXContent { private static final Logger log = LogManager.getLogger(DefaultIocStoreConfig.class); public static final String DEFAULT_FIELD = "default"; - public static final String IOC_MAP = "ioc_map"; + public static final String IOC_TO_INDEX_DETAILS_FIELD = "ioc_to_index_details"; + private final List iocToIndexDetails; - // Maps the IOC types to the list of index/alias names - private final Map> iocMapStore; - - public DefaultIocStoreConfig(Map> iocMapStore) { - this.iocMapStore = iocMapStore; + public DefaultIocStoreConfig(List iocToIndexDetails) { + this.iocToIndexDetails = iocToIndexDetails; } public DefaultIocStoreConfig(StreamInput sin) throws IOException { - this.iocMapStore = sin.readMapOfLists(StreamInput::readString, StreamInput::readString); + this.iocToIndexDetails = Collections.unmodifiableList(sin.readList(IocToIndexDetails::new)); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeMapOfLists(iocMapStore, StreamOutput::writeString, StreamOutput::writeString); + out.writeCollection(iocToIndexDetails); } public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject() .field(DEFAULT_FIELD); builder.startObject() - .field(IOC_MAP, iocMapStore); + .field(IOC_TO_INDEX_DETAILS_FIELD, iocToIndexDetails); builder.endObject(); builder.endObject(); return builder; } public static DefaultIocStoreConfig parse(XContentParser xcp) throws IOException { - Map> iocMapStore = null; + List iocToIndexDetails = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); - switch (fieldName) { case DEFAULT_FIELD: break; - case IOC_MAP: + case IOC_TO_INDEX_DETAILS_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { - iocMapStore = null; + iocToIndexDetails = null; } else { - iocMapStore = xcp.map(HashMap::new, p -> { - List indices = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); - while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - indices.add(xcp.text()); - } - return indices; - }); + iocToIndexDetails = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocToIndexDetails.add(IocToIndexDetails.parse(xcp)); + } } break; default: xcp.skipChildren(); } } - return new DefaultIocStoreConfig(iocMapStore); + return new DefaultIocStoreConfig(iocToIndexDetails); } @Override @@ -88,8 +82,92 @@ public String name() { return DEFAULT_FIELD; } - public Map> getIocMapStore() { - return iocMapStore; + public List getIocToIndexDetails() { + return iocToIndexDetails; } + public static class IocToIndexDetails implements Writeable, ToXContent { + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String INDEX_PATTERN_FIELD = "index_pattern"; + public static final String ACTIVE_INDEX_FIELD = "active_index"; + private final IOCType iocType; + private final String indexPattern; + private final String activeIndex; + + public IocToIndexDetails(IOCType iocType, String indexPattern, String activeIndex) { + this.iocType = iocType; + this.indexPattern = indexPattern; + this.activeIndex = activeIndex; + } + + public IocToIndexDetails(StreamInput sin) throws IOException { + this(sin.readEnum(IOCType.class), + sin.readString(), + sin.readString()); + } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(iocType); + out.writeString(indexPattern); + out.writeString(activeIndex); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOC_TYPE_FIELD, iocType) + .field(INDEX_PATTERN_FIELD, indexPattern) + .field(ACTIVE_INDEX_FIELD, activeIndex) + .endObject(); + } + + public static IocToIndexDetails parse(XContentParser xcp) throws IOException { + IOCType iocType = null; + String indexPattern = null; + String activeIndex = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPE_FIELD: + iocType = toIocType(xcp.text()); + break; + case INDEX_PATTERN_FIELD: + indexPattern = xcp.text(); + break; + case ACTIVE_INDEX_FIELD: + activeIndex = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocToIndexDetails(iocType, indexPattern, activeIndex); + } + + public static IOCType toIocType(String name) { + try { + return IOCType.fromString(name); + } catch (IllegalArgumentException e) { + log.error("Invalid Ioc type, cannot be parsed.", e); + return null; + } + } + + public IOCType getIocType() { + return iocType; + } + + public String getIndexPattern() { + return indexPattern; + } + + public String getActiveIndex() { + return activeIndex; + } + + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 11f6367f2..51b909334 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -58,6 +58,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ public static final String CREATED_AT_FIELD = "created_at"; public static final String SOURCE_FIELD = "source"; public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_FOR_SCAN_FIELD = "enabled_for_scan"; public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; public static final String SCHEDULE_FIELD = "schedule"; public static final String STATE_FIELD = "state"; @@ -87,10 +88,11 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ private Boolean isEnabled; private IocStoreConfig iocStoreConfig; private List iocTypes; + private final boolean enabledForScan; public SATIFSourceConfig(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes) { + Boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -100,6 +102,7 @@ public SATIFSourceConfig(String id, Long version, String name, String format, So this.createdByUser = createdByUser; this.createdAt = createdAt != null ? createdAt : Instant.now(); this.source = source; + this.enabledForScan = enabledForScan; if (isEnabled && enabledTime == null) { this.enabledTime = Instant.now(); @@ -140,7 +143,8 @@ public SATIFSourceConfig(StreamInput sin) throws IOException { sin.readBoolean() ? new User(sin) : null, // last refreshed user sin.readBoolean(), // is enabled IocStoreConfig.readFrom(sin), // ioc map store - sin.readStringList() // ioc types + sin.readStringList(), // ioc types + sin.readBoolean() // enabled for scan ); } @@ -181,6 +185,7 @@ public void writeTo(final StreamOutput out) throws IOException { } iocStoreConfig.writeTo(out); out.writeStringCollection(iocTypes); + out.writeBoolean(enabledForScan); } @Override @@ -242,6 +247,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); } builder.field(ENABLED_FIELD, isEnabled); + builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); builder.field(IOC_STORE_FIELD, iocStoreConfig); builder.field(IOC_TYPES_FIELD, iocTypes); builder.endObject(); @@ -284,6 +290,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio Instant lastRefreshedTime = null; User lastRefreshedUser = null; Boolean isEnabled = null; + boolean enabledForScan = true; IocStoreConfig iocStoreConfig = null; List iocTypes = new ArrayList<>(); @@ -399,6 +406,9 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; + case ENABLED_FOR_SCAN_FIELD: + enabledForScan = xcp.booleanValue(); + break; case IOC_STORE_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { iocStoreConfig = null; @@ -442,7 +452,8 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio lastRefreshedUser, isEnabled, iocStoreConfig, - iocTypes + iocTypes, + enabledForScan ); } @@ -477,7 +488,7 @@ public static RefreshType toRefreshType(String stateName) { private IocStoreConfig newIocStoreConfig(String storeType) { switch (storeType) { case "default": - return new DefaultIocStoreConfig(new HashMap<>()); + return new DefaultIocStoreConfig(new ArrayList<>()); default: throw new IllegalStateException("Unexpected store type"); } @@ -645,7 +656,13 @@ public List getIocTypes() { return iocTypes; } + @Override public void setIocTypes(List iocTypes) { this.iocTypes = iocTypes; } + + @Override + public boolean isEnabledForScan() { + return this.enabledForScan; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index 887cad680..ee1dc4f6f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -59,6 +59,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou public static final String CREATED_AT_FIELD = "created_at"; public static final String SOURCE_FIELD = "source"; public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_FOR_SCAN_FIELD = "enabled_for_scan"; public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; public static final String SCHEDULE_FIELD = "schedule"; public static final String STATE_FIELD = "state"; @@ -86,6 +87,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou public User lastRefreshedUser; private Boolean isEnabled; private List iocTypes; + private final boolean enabledForScan; public SATIFSourceConfigDto(SATIFSourceConfig saTifSourceConfig) { this.id = saTifSourceConfig.getId(); @@ -106,6 +108,7 @@ public SATIFSourceConfigDto(SATIFSourceConfig saTifSourceConfig) { this.lastRefreshedUser = saTifSourceConfig.getLastRefreshedUser(); this.isEnabled = saTifSourceConfig.isEnabled(); this.iocTypes = saTifSourceConfig.getIocTypes(); + this.enabledForScan = saTifSourceConfig.isEnabledForScan(); } private List convertToIocDtos(List stix2IocList) { @@ -116,7 +119,7 @@ private List convertToIocDtos(List stix2IocList) { public SATIFSourceConfigDto(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, List iocTypes) { + Boolean isEnabled, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -143,6 +146,7 @@ public SATIFSourceConfigDto(String id, Long version, String name, String format, this.lastRefreshedUser = lastRefreshedUser; this.isEnabled = isEnabled; this.iocTypes = iocTypes; + this.enabledForScan = enabledForScan; } public SATIFSourceConfigDto(StreamInput sin) throws IOException { @@ -164,7 +168,8 @@ public SATIFSourceConfigDto(StreamInput sin) throws IOException { sin.readOptionalInstant(), // last refreshed time sin.readBoolean() ? new User(sin) : null, // last refreshed user sin.readBoolean(), // is enabled - sin.readStringList() // ioc types + sin.readStringList(), // ioc types + sin.readBoolean() ); } @@ -201,6 +206,7 @@ public void writeTo(final StreamOutput out) throws IOException { } out.writeBoolean(isEnabled); out.writeStringCollection(iocTypes); + out.writeBoolean(enabledForScan); } @Override @@ -261,6 +267,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); } builder.field(ENABLED_FIELD, isEnabled); + builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); builder.field(IOC_TYPES_FIELD, iocTypes); builder.endObject(); builder.endObject(); @@ -300,6 +307,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver User lastRefreshedUser = null; Boolean isEnabled = null; List iocTypes = new ArrayList<>(); + boolean enabledForScan = true; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -409,6 +417,10 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; + + case ENABLED_FOR_SCAN_FIELD: + enabledForScan = xcp.booleanValue(); + break; case IOC_TYPES_FIELD: XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { @@ -444,7 +456,8 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver lastRefreshedTime, lastRefreshedUser, isEnabled, - iocTypes + iocTypes, + enabledForScan ); } @@ -639,4 +652,8 @@ public void setIocTypes(List iocTypes) { public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { return new SATIFSourceConfigDto(sin); } + + public boolean isEnabledForScan() { + return enabledForScan; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java index d399a1b08..dae00034a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -71,4 +71,5 @@ public interface TIFSourceConfig { public void setIocTypes(List iocTypes); + boolean isEnabledForScan(); } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index fd164224d..1d9a71a2b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -3,13 +3,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.delete.DeleteResponse; -import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -18,18 +17,16 @@ import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.rest.RestRequest; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; @@ -41,17 +38,17 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; -import org.opensearch.securityanalytics.util.IndexUtils; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; - import java.util.stream.Collectors; import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; @@ -197,7 +194,24 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, stix2IOCFetchService.downloadAndIndexIOCs(saTifSourceConfig, actionListener); break; case IOC_UPLOAD: - stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, stix2IOCList, actionListener); + List validStix2IocList = new ArrayList<>(); + // If the IOC received is not a type listed for the config, do not add it to the queue + for (STIX2IOC stix2IOC : stix2IOCList) { + if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().name())) { + validStix2IocList.add(stix2IOC); + } else { + log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}", + stix2IOC.getType().name(), saTifSourceConfig.getId(), + stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue() + ); + } + } + if (validStix2IocList.isEmpty()) { + log.error("No supported IOCs to index"); + actionListener.onFailure(new OpenSearchStatusException("No compatible Iocs were uploaded for config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)); + return; + } + stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener); break; } } @@ -222,10 +236,8 @@ public void searchTIFSourceConfigs( final ActionListener listener ) { try { - SearchRequest searchRequest = getSearchRequest(searchSourceBuilder); - // convert search response to threat intel source config dtos - saTifSourceConfigService.searchTIFSourceConfigs(searchRequest, ActionListener.wrap( + saTifSourceConfigService.searchTIFSourceConfigs(searchSourceBuilder, ActionListener.wrap( searchResponse -> { for (SearchHit hit : searchResponse.getHits()) { XContentParser xcp = XContentType.JSON.xContent().createParser( @@ -248,33 +260,6 @@ public void searchTIFSourceConfigs( } } - private static SearchRequest getSearchRequest(SearchSourceBuilder searchSourceBuilder) { - - // update search source builder - searchSourceBuilder.seqNoAndPrimaryTerm(true); - searchSourceBuilder.version(true); - - // construct search request - SearchRequest searchRequest = new SearchRequest().source(searchSourceBuilder); - searchRequest.indices(SecurityAnalyticsPlugin.JOB_INDEX_NAME); - searchRequest.preference(Preference.PRIMARY_FIRST.type()); - - BoolQueryBuilder boolQueryBuilder; - - if (searchRequest.source().query() == null) { - boolQueryBuilder = new BoolQueryBuilder(); - } else { - boolQueryBuilder = QueryBuilders.boolQuery().must(searchRequest.source().query()); - } - - BoolQueryBuilder bqb = new BoolQueryBuilder(); - bqb.should().add(new BoolQueryBuilder().must(QueryBuilders.existsQuery("source_config"))); - - boolQueryBuilder.filter(bqb); - searchRequest.source().query(boolQueryBuilder); - return searchRequest; - } - public void updateIocAndTIFSourceConfig( final SATIFSourceConfigDto saTifSourceConfigDto, final LockModel lock, @@ -347,39 +332,56 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap( downloadAndSaveIocsResponse -> { - // delete the old ioc index created with the source config - String type = updatedSaTifSourceConfig.getIocTypes().get(0); - Map> iocToAliasMap = ((DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig()).getIocMapStore(); - List iocIndices = iocToAliasMap.get(type); - List indicesToDelete = new ArrayList<>(); - String alias = getIocIndexAlias(updatedSaTifSourceConfig.getId()); - String writeIndex = IndexUtils.getWriteIndex(alias, clusterService.state()); - for (String index: iocIndices) { - if (index.equals(writeIndex) == false && index.equals(alias) == false) { - indicesToDelete.add(index); - } + Set iocIndexPatterns = new HashSet<>(); + if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + // get all the index patterns + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); } - // delete the old indices - saTifSourceConfigService.deleteAllIocIndices(indicesToDelete, true, null); - // remove all indices from the store config from above list for all types - for (String iocType : updatedSaTifSourceConfig.getIocTypes()) { - iocToAliasMap.get(iocType).removeAll(indicesToDelete); - } + saTifSourceConfigService.getClusterState(ActionListener.wrap( + clusterStateResponse -> { + List iocTypes = updatedSaTifSourceConfig.getIocTypes(); + IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig(); + Set activeIndices = new HashSet<>(); + Set indicesToDelete = new HashSet<>(); + + if (iocStoreConfig instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) iocStoreConfig; + Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + + // remove ioc types not specified in list + defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name())); - updatedSaTifSourceConfig.setIocStoreConfig(new DefaultIocStoreConfig(iocToAliasMap)); - markSourceConfigAsAction( - updatedSaTifSourceConfig, - TIFJobState.AVAILABLE, - ActionListener.wrap( - saTifSourceConfigResponse -> { - SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse); - listener.onResponse(returnedSaTifSourceConfigDto); - }, e -> { - log.error("Failed to index threat intel source config with id [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(e); + // get the active indices + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); + + for (String index : concreteIndices) { + if (false == activeIndices.contains(index)) { + indicesToDelete.add(index); + } } - )); + } + + // delete the old indices + saTifSourceConfigService.deleteAllIocIndices(indicesToDelete, true, null); + markSourceConfigAsAction( + updatedSaTifSourceConfig, + TIFJobState.AVAILABLE, + ActionListener.wrap( + saTifSourceConfigResponse -> { + SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse); + listener.onResponse(returnedSaTifSourceConfigDto); + }, e -> { + log.error("Failed to index threat intel source config with id [{}]", updatedSaTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to get the cluster metadata"); + listener.onFailure(e); + } + ), iocIndexPatterns.toArray(new String[0])); }, e -> { log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId()); @@ -458,7 +460,13 @@ private void downloadAndSaveIocsToRefresh(ActionListener l // delete old IOCs and update the source config deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap( newIocStoreConfig -> { - updatedSourceConfig.setIocStoreConfig(newIocStoreConfig); + List iocTypes = updatedSourceConfig.getIocTypes(); + if (newIocStoreConfig instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig; + // remove ioc types not specified in list + defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name())); + updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig); + } // Update source config as succeeded, change state back to available markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( r -> { @@ -504,7 +512,7 @@ public void deleteTIFSourceConfig( // Check if all threat intel monitors are deleted saTifSourceConfigService.checkAndEnsureThreatIntelMonitorsDeleted(ActionListener.wrap( isDeleted -> { - onDeleteThreatIntelMonitors(saTifSourceConfigId, listener, saTifSourceConfig, isDeleted); + deleteAllIocsAndSourceConfig(saTifSourceConfigId, listener, saTifSourceConfig, isDeleted); }, e -> { log.error("Failed to check if all threat intel monitors are deleted or if multiple threat intel source configs exist"); listener.onFailure(e); @@ -522,7 +530,7 @@ public void deleteTIFSourceConfig( } /** - * Deletes the old ioc indices based on retention age and number of indices per alias + * Deletes the old ioc indices based on retention age and number of indices per index pattern * * @param saTifSourceConfig * @param listener @@ -531,66 +539,63 @@ public void deleteOldIocIndices( final SATIFSourceConfig saTifSourceConfig, ActionListener listener ) { - Map> iocToAliasMap = ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore(); - - // Grabbing the first ioc type since all the indices are stored in one index - String type = saTifSourceConfig.getIocTypes().get(0); - String alias = getIocIndexAlias(saTifSourceConfig.getId()); - List concreteIndices = new ArrayList<>(iocToAliasMap.get(type)); - concreteIndices.remove(alias); + Set activeIndices = new HashSet<>(); + IocStoreConfig iocStoreConfig = saTifSourceConfig.getIocStoreConfig(); + Set iocIndexPatterns = new HashSet<>(); + if (iocStoreConfig instanceof DefaultIocStoreConfig) { + // get the active indices + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); + // get all the index patterns + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } saTifSourceConfigService.getClusterState(ActionListener.wrap( clusterStateResponse -> { - List indicesToDeleteByAge = getIocIndicesToDeleteByAge(clusterStateResponse.getState(), alias); + Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + List indicesToDeleteByAge = getIocIndicesToDeleteByAge(clusterStateResponse.getState(), activeIndices); List indicesToDeleteBySize = getIocIndicesToDeleteBySize( clusterStateResponse.getState(), - iocToAliasMap.get(type).size(), indicesToDeleteByAge.size(), - alias, + activeIndices, concreteIndices); - List iocIndicesToDelete = new ArrayList<>(); + Set iocIndicesToDelete = new HashSet<>(); iocIndicesToDelete.addAll(indicesToDeleteByAge); iocIndicesToDelete.addAll(indicesToDeleteBySize); // delete the indices saTifSourceConfigService.deleteAllIocIndices(iocIndicesToDelete, true, null); - // update source config - saTifSourceConfig.getIocTypes() - .stream() - .forEach(iocType -> iocToAliasMap.get(iocType).removeAll(iocIndicesToDelete)); - - // return source config - listener.onResponse(new DefaultIocStoreConfig(iocToAliasMap)); - }, e-> { + // return store config + listener.onResponse(iocStoreConfig); + }, e -> { log.error("Failed to get the cluster metadata"); listener.onFailure(e); } - ), concreteIndices.toArray(new String[0])); + ), iocIndexPatterns.toArray(new String[0])); } /** * Helper function to retrieve a list of IOC indices to delete based on retention age * * @param clusterState - * @param alias + * @param activeIndices * @return indicesToDelete */ private List getIocIndicesToDeleteByAge( ClusterState clusterState, - String alias + Set activeIndices ) { List indicesToDelete = new ArrayList<>(); - String writeIndex = IndexUtils.getWriteIndex(alias, clusterState); Long maxRetentionPeriod = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD).millis(); for (IndexMetadata indexMetadata : clusterState.metadata().indices().values()) { Long creationTime = indexMetadata.getCreationDate(); if ((Instant.now().toEpochMilli() - creationTime) > maxRetentionPeriod) { String indexToDelete = indexMetadata.getIndex().getName(); - // ensure index is not the current write index - if (indexToDelete.equals(writeIndex) == false) { + // ensure index is not the current active index + if (activeIndices.contains(indexToDelete) == false) { indicesToDelete.add(indexToDelete); } } @@ -600,27 +605,24 @@ private List getIocIndicesToDeleteByAge( /** - * Helper function to retrieve a list of IOC indices to delete based on number of indices associated with alias + * Helper function to retrieve a list of IOC indices to delete based on number of indices associated with the index pattern + * * @param clusterState - * @param totalNumIndicesAndAlias * @param totalNumIndicesDeleteByAge - * @param alias + * @param activeIndices * @param concreteIndices * @return */ private List getIocIndicesToDeleteBySize( ClusterState clusterState, - Integer totalNumIndicesAndAlias, Integer totalNumIndicesDeleteByAge, - String alias, - List concreteIndices + Set activeIndices, + Set concreteIndices ) { - Integer numIndicesToDelete = numOfIndicesToDelete(totalNumIndicesAndAlias - 1, totalNumIndicesDeleteByAge); // subtract to account for alias + Integer numIndicesToDelete = numOfIndicesToDelete(concreteIndices.size(), totalNumIndicesDeleteByAge); List indicesToDelete = new ArrayList<>(); if (numIndicesToDelete > 0) { - String writeIndex = IndexUtils.getWriteIndex(alias, clusterState); - // store indices and creation date in map Map indexToAgeMap = new LinkedHashMap<>(); final SortedMap lookup = clusterState.getMetadata().getIndicesLookup(); @@ -638,12 +640,12 @@ private List getIocIndicesToDeleteBySize( // ensure range is not out of bounds int endIndex = totalNumIndicesDeleteByAge + numIndicesToDelete; - endIndex = Math.min(endIndex, totalNumIndicesAndAlias); + endIndex = Math.min(endIndex, concreteIndices.size()); // grab names of indices from totalNumIndicesDeleteByAge to totalNumIndicesDeleteByAge + numIndicesToDelete for (int i = totalNumIndicesDeleteByAge; i < endIndex; i++) { - // ensure index is not the current write index - if (false == sortedList.get(i).getKey().equals(writeIndex)) { + // ensure index is not a current active index + if (false == activeIndices.contains(sortedList.get(i).getKey())) { indicesToDelete.add(sortedList.get(i).getKey()); } } @@ -652,21 +654,22 @@ private List getIocIndicesToDeleteBySize( } /** - * Helper function to determine how many indices should be deleted based on setting for number of indices per alias + * Helper function to determine how many indices should be deleted based on setting for number of indices per index pattern + * * @param totalNumIndices * @param totalNumIndicesDeleteByAge * @return */ private Integer numOfIndicesToDelete(Integer totalNumIndices, Integer totalNumIndicesDeleteByAge) { - Integer maxIndicesPerAlias = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS); + Integer maxIndicesPerIndexPattern = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN); Integer numIndicesAfterDeletingByAge = totalNumIndices - totalNumIndicesDeleteByAge; - if (numIndicesAfterDeletingByAge > maxIndicesPerAlias) { - return numIndicesAfterDeletingByAge - maxIndicesPerAlias; + if (numIndicesAfterDeletingByAge > maxIndicesPerIndexPattern) { + return numIndicesAfterDeletingByAge - maxIndicesPerIndexPattern; } return 0; } - private void onDeleteThreatIntelMonitors(String saTifSourceConfigId, ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { + private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { if (isDeleted == false) { listener.onFailure(new IllegalArgumentException("All threat intel monitors need to be deleted before deleting last threat intel source config")); } else { @@ -676,27 +679,37 @@ private void onDeleteThreatIntelMonitors(String saTifSourceConfigId, ActionListe TIFJobState.DELETING, ActionListener.wrap( updateSaTifSourceConfigResponse -> { - String type = updateSaTifSourceConfigResponse.getIocTypes().get(0); - DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) updateSaTifSourceConfigResponse.getIocStoreConfig(); - List indicesWithoutAlias = new ArrayList<>(iocStoreConfig.getIocMapStore().get(type)); - indicesWithoutAlias.remove(getIocIndexAlias(updateSaTifSourceConfigResponse.getId())); - saTifSourceConfigService.deleteAllIocIndices(indicesWithoutAlias, false, ActionListener.wrap( - r -> { - log.debug("Successfully deleted all ioc indices"); - saTifSourceConfigService.deleteTIFSourceConfig(updateSaTifSourceConfigResponse, ActionListener.wrap( - deleteResponse -> { - log.debug("Successfully deleted threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); - listener.onResponse(deleteResponse); + Set iocIndexPatterns = new HashSet<>(); + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + // get all the index patterns + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } + saTifSourceConfigService.getClusterState(ActionListener.wrap( + clusterStateResponse -> { + Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + saTifSourceConfigService.deleteAllIocIndices(concreteIndices, false, ActionListener.wrap( + r -> { + log.debug("Successfully deleted all ioc indices"); + saTifSourceConfigService.deleteTIFSourceConfig(updateSaTifSourceConfigResponse, ActionListener.wrap( + deleteResponse -> { + log.debug("Successfully deleted threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); + listener.onResponse(deleteResponse); + }, e -> { + log.error("Failed to delete threat intel source config [{}]", saTifSourceConfigId); + listener.onFailure(e); + } + )); }, e -> { - log.error("Failed to delete threat intel source config [{}]", saTifSourceConfigId); + log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId()); listener.onFailure(e); } )); }, e -> { - log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId()); + log.error("Failed to get the cluster metadata"); listener.onFailure(e); } - )); + ), iocIndexPatterns.toArray(new String[0])); }, e -> { log.error("Failed to update threat intel source config with state as {}", TIFJobState.DELETING); listener.onFailure(e); @@ -706,11 +719,12 @@ private void onDeleteThreatIntelMonitors(String saTifSourceConfigId, ActionListe } public void markSourceConfigAsAction(final SATIFSourceConfig saTifSourceConfig, TIFJobState state, ActionListener actionListener) { + TIFJobState previousState = saTifSourceConfig.getState(); saTifSourceConfig.setState(state); try { internalUpdateTIFSourceConfig(saTifSourceConfig, actionListener); } catch (Exception e) { - log.error("Failed to mark threat intel source config as {} for [{}]", state, saTifSourceConfig.getId(), e); + log.error("Failed to mark threat intel source config from {} to {} for [{}]", previousState, state, saTifSourceConfig.getId(), e); actionListener.onFailure(e); } } @@ -725,6 +739,10 @@ public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceCo IocStoreConfig iocStoreConfig, TIFJobState state, User createdByUser) { + + // remove duplicates from iocTypes + Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); + return new SATIFSourceConfig( saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), @@ -744,11 +762,15 @@ public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceCo saTifSourceConfigDto.getLastRefreshedUser(), saTifSourceConfigDto.isEnabled(), iocStoreConfig, - saTifSourceConfigDto.getIocTypes() + new ArrayList<>(iocTypes), + saTifSourceConfigDto.isEnabledForScan() ); } private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSourceConfigDto, SATIFSourceConfig saTifSourceConfig) { + // remove duplicates from iocTypes + Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); + return new SATIFSourceConfig( saTifSourceConfig.getId(), saTifSourceConfig.getVersion(), @@ -768,7 +790,8 @@ private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSour saTifSourceConfig.getLastRefreshedUser(), saTifSourceConfigDto.isEnabled(), saTifSourceConfig.getIocStoreConfig(), - saTifSourceConfigDto.getIocTypes() + new ArrayList<>(iocTypes), + saTifSourceConfigDto.isEnabledForScan() ); } @@ -780,5 +803,4 @@ public List convertToIocs(List stix2IocDtoList, String na .map(dto -> new STIX2IOC(dto, id, name)) .collect(Collectors.toList()); } - } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index e733edbf4..14aa45bb1 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -26,8 +26,7 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; -import org.opensearch.client.Request; -import org.opensearch.client.Response; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -49,6 +48,7 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; @@ -64,15 +64,19 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.ENABLED_FOR_SCAN_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.STATE_FIELD; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; @@ -161,7 +165,8 @@ private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig saTif saTifSourceConfig.getLastRefreshedUser(), saTifSourceConfig.isEnabled(), saTifSourceConfig.getIocStoreConfig(), - saTifSourceConfig.getIocTypes() + saTifSourceConfig.getIocTypes(), + saTifSourceConfig.isEnabledForScan() ); } @@ -246,9 +251,11 @@ public void getTIFSourceConfig( } public void searchTIFSourceConfigs( - final SearchRequest searchRequest, + final SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener ) { + SearchRequest searchRequest = getSearchRequest(searchSourceBuilder); + // Check to make sure the job index exists if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist")); @@ -282,6 +289,33 @@ public void searchTIFSourceConfigs( ); } + private static SearchRequest getSearchRequest(SearchSourceBuilder searchSourceBuilder) { + + // update search source builder + searchSourceBuilder.seqNoAndPrimaryTerm(true); + searchSourceBuilder.version(true); + + // construct search request + SearchRequest searchRequest = new SearchRequest().source(searchSourceBuilder); + searchRequest.indices(SecurityAnalyticsPlugin.JOB_INDEX_NAME); + searchRequest.preference(Preference.PRIMARY_FIRST.type()); + + BoolQueryBuilder boolQueryBuilder; + + if (searchRequest.source().query() == null) { + boolQueryBuilder = new BoolQueryBuilder(); + } else { + boolQueryBuilder = QueryBuilders.boolQuery().must(searchRequest.source().query()); + } + + BoolQueryBuilder bqb = new BoolQueryBuilder(); + bqb.should().add(new BoolQueryBuilder().must(QueryBuilders.existsQuery("source_config"))); + + boolQueryBuilder.filter(bqb); + searchRequest.source().query(boolQueryBuilder); + return searchRequest; + } + // Update TIF source config public void updateTIFSourceConfig( SATIFSourceConfig saTifSourceConfig, @@ -341,7 +375,7 @@ public void deleteTIFSourceConfig( )); } - public void deleteAllIocIndices(List indicesToDelete, Boolean backgroundJob, ActionListener listener) { + public void deleteAllIocIndices(Set indicesToDelete, Boolean backgroundJob, ActionListener listener) { if (indicesToDelete.isEmpty() == false) { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); client.admin().indices().delete( @@ -366,7 +400,7 @@ public void deleteAllIocIndices(List indicesToDelete, Boolean background } } - private void deleteIocIndex(List indicesToDelete, Boolean backgroundJob, ActionListener listener) { + private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, ActionListener listener) { for (String index : indicesToDelete) { final DeleteIndexRequest singleDeleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); client.admin().indices().delete( @@ -397,8 +431,7 @@ private void deleteIocIndex(List indicesToDelete, Boolean backgroundJob, public void getClusterState( final ActionListener actionListener, - String... indices) - { + String... indices) { ClusterStateRequest clusterStateRequest = new ClusterStateRequest() .clear() .indices(indices) @@ -486,13 +519,22 @@ public void checkAndEnsureThreatIntelMonitorsDeleted( } + /** + * Returns a map of ioc type to a list of active indices + * + * @param listener + */ public void getIocTypeToIndices(ActionListener>> listener) { SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.termQuery(getEnabledForScanFieldName(), true)); + String stateFieldName = getStateFieldName(); - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery() .should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString())); - queryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING)); + stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING)); + queryBuilder.must(stateQueryBuilder); searchRequest.source().query(queryBuilder); client.search(searchRequest, ActionListener.wrap( @@ -506,12 +548,11 @@ public void getIocTypeToIndices(ActionListener>> listen SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); - Map> iocTypeToIndices = iocStoreConfig.getIocMapStore(); - for (String iocType : iocTypeToIndices.keySet()) { - if (iocTypeToIndices.get(iocType).isEmpty()) - continue; - List strings = cumulativeIocTypeToIndices.computeIfAbsent(iocType, k -> new ArrayList<>()); - strings.addAll(iocTypeToIndices.get(iocType)); + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) { + String activeIndex = iocToindexDetails.getActiveIndex(); + IOCType iocType = iocToindexDetails.getIocType(); + List strings = cumulativeIocTypeToIndices.computeIfAbsent(iocType.toString(), k -> new ArrayList<>()); + strings.add(activeIndex); } } } @@ -527,4 +568,18 @@ public void getIocTypeToIndices(ActionListener>> listen public static String getStateFieldName() { return String.format("%s.%s", SOURCE_CONFIG_FIELD, STATE_FIELD); } + + + public static String getEnabledForScanFieldName() { + return String.format("%s.%s", SOURCE_CONFIG_FIELD, ENABLED_FOR_SCAN_FIELD); + } + + public static Set getConcreteIndices(ClusterStateResponse clusterStateResponse) { + Set concreteIndices = new HashSet<>(); + Collection values = clusterStateResponse.getState().metadata().indices().values(); + for (IndexMetadata metadata : values) { + concreteIndices.add(metadata.getIndex().getName()); + } + return concreteIndices; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java index 23d0b3a0d..d046a35e5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -62,7 +62,7 @@ protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, Act return; } - this.threadPool.getThreadContext().stashContext(); // TODO: sync up with @deysubho about thread context + this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( r -> { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 4abd3750c..132725d71 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -41,9 +41,10 @@ import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; -import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -56,7 +57,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; import static org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService.getStateFieldName; @@ -71,12 +71,14 @@ public class TransportListIOCsAction extends HandledTransportAction configIds = request.getFeedIds() == null ? Collections.emptyList() : request.getFeedIds(); - transportSearchTIFSourceConfigsAction.execute(new SASearchTIFSourceConfigsRequest(getFeedsSearchSourceBuilder(configIds)), + saTifSourceConfigService.searchTIFSourceConfigs(getFeedsSearchSourceBuilder(configIds), ActionListener.wrap( searchResponse -> { List iocIndices = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits().getHits()) { - String iocIndexAlias = getIocIndexAlias(hit.getId()); - if (IndexUtils.isAlias(iocIndexAlias, clusterService.state())) { - String writeIndex = IndexUtils.getWriteIndex(iocIndexAlias, clusterService.state()); - if (writeIndex != null) - iocIndices.add(writeIndex); + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); + if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails: iocStoreConfig.getIocToIndexDetails()) { + String writeIndex = iocToindexDetails.getActiveIndex(); + if (writeIndex != null) { + iocIndices.add(writeIndex); + } + } } } if (iocIndices.isEmpty()) { diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 0f3b2fe81..f437efe6f 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -128,6 +128,9 @@ "refresh_type": { "type": "keyword" }, + "enabled_for_scan": { + "type": "boolean" + }, "last_refreshed_time": { "type": "date", "format": "strict_date_time||epoch_millis" @@ -176,19 +179,16 @@ "properties": { "default": { "properties": { - "ioc_map": { + "ioc_to_index_details": { "properties": { - "domain-name": { - "type": "text" - }, - "hashes": { - "type": "text" + "ioc_type": { + "type": "keyword" }, - "ipv4_addr": { - "type": "text" + "index_pattern": { + "type": "keyword" }, - "ipv6_addr": { - "type": "text" + "active_index": { + "type": "keyword" } } } diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 37659b36f..92bed3a30 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -246,7 +246,7 @@ public static CorrelationRule randomCorrelationRule(String name) { } public static CorrelationRule randomCorrelationRuleWithTrigger(String name) { - name = name.isEmpty()? ">": name; + name = name.isEmpty() ? ">" : name; List actions = new ArrayList(); CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-123", "Trigger 1", "high", actions); return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, @@ -2823,7 +2823,8 @@ public static SATIFSourceConfigDto randomSATIFSourceConfigDto( lastRefreshedTime, lastRefreshedUser, isEnabled, - iocTypes + iocTypes, + true ); } @@ -2887,9 +2888,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig( schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); } if (iocStoreConfig == null) { - Map> iocMapStore = new HashMap<>(); - iocMapStore.put("ip", List.of("index_name")); - iocStoreConfig = new DefaultIocStoreConfig(iocMapStore); + iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.domain_name, "indexPattern", "writeIndex"))); } if (iocTypes == null) { iocTypes = List.of("ip"); @@ -2914,7 +2913,8 @@ public static SATIFSourceConfig randomSATIFSourceConfig( lastRefreshedUser, isEnabled, iocStoreConfig, - iocTypes + iocTypes, + true ); } } diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java index 1b89d906f..9acb3da4e 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java @@ -53,7 +53,8 @@ public void testStreamInOut() throws IOException { Instant.now(), null, false, - iocTypes + iocTypes, + true ); SAGetTIFSourceConfigResponse response = new SAGetTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto); diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java index 45dd831ef..f720099bf 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java @@ -49,7 +49,8 @@ public void testIndexTIFSourceConfigPostResponse() throws IOException { Instant.now(), null, false, - iocTypes + iocTypes, + true ); SAIndexTIFSourceConfigResponse response = new SAIndexTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto); diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java index 4f185a775..61f7ecf07 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java @@ -75,7 +75,9 @@ private void assertEqualsSaTifSourceConfigs(SATIFSourceConfig saTifSourceConfig, assertEquals(saTifSourceConfig.isEnabled(), newSaTifSourceConfig.isEnabled()); DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); DefaultIocStoreConfig newIocStoreConfig = (DefaultIocStoreConfig) newSaTifSourceConfig.getIocStoreConfig(); - assertEquals(iocStoreConfig.getIocMapStore().keySet(), newIocStoreConfig.getIocMapStore().keySet()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIocType(), newIocStoreConfig.getIocToIndexDetails().get(0).getIocType()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern(), newIocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex(), newIocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex()); assertEquals(saTifSourceConfig.getIocTypes(), newSaTifSourceConfig.getIocTypes()); } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 3abbf79d5..7624a746a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; /** * The following system parameters must be specified to successfully run these tests: @@ -135,7 +136,7 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int Instant.now(), null, true, - iocTypes + iocTypes, true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(201, response.getStatusLine().getStatusCode()); @@ -229,7 +230,7 @@ public void testGetSATIFSourceConfigById() throws IOException { Instant.now(), null, true, - iocTypes + iocTypes, true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); @@ -295,7 +296,7 @@ public void testDeleteSATIFSourceConfig() throws IOException { Instant.now(), null, true, - iocTypes + iocTypes, true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); @@ -364,7 +365,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept Instant.now(), null, true, - iocTypes + iocTypes, true ); // Confirm test feed was created successfully @@ -387,7 +388,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept }, 240, TimeUnit.SECONDS); // Confirm IOCs were ingested to system index for the feed - String indexName = STIX2IOCFeedStore.getIocIndexAlias(createdId); + String indexName = getAllIocIndexPatternById(createdId); String request = "{\n" + " \"query\" : {\n" + " \"match_all\":{\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 7c2b50d48..cd651a012 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -32,6 +32,7 @@ import java.util.Map; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; public class SourceConfigWithoutS3RestApiIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(SourceConfigWithoutS3RestApiIT.class); @@ -77,7 +78,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { null, null, enabled, - iocTypes + iocTypes, true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); @@ -101,7 +102,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { Assert.assertEquals(1, hits.size()); // ensure same number of iocs got indexed - String indexName = STIX2IOCFeedStore.getIocIndexAlias(createdId); + String indexName = getAllIocIndexPatternById(createdId); hits = executeSearch(indexName, request); Assert.assertEquals(iocs.size(), hits.size()); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 1f3bb7a74..bec504072 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -47,10 +47,11 @@ public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { public void indexSourceConfigsAndIocs(int num, List iocVals) throws IOException { for (int i = 0; i < num; i++) { String configId = "id" + i; - String iocIndexName = ".opensearch-sap-ioc-" + configId; - indexTifSourceConfig(num, configId, iocIndexName, i); + String iocActiveIndex = ".opensearch-sap-ioc-" + configId + Instant.now().toEpochMilli(); + String indexPattern = ".opensearch-sap-ioc-" + configId; + indexTifSourceConfig(num, configId, indexPattern, iocActiveIndex, i); for (int i1 = 0; i1 < iocVals.size(); i1++) { - indexIocs(iocVals, iocIndexName, i1, configId); + indexIocs(iocVals, iocActiveIndex, i1, configId); } } } @@ -77,7 +78,7 @@ private void indexIocs(List iocVals, String iocIndexName, int i1, String assertEquals(searchHits.size(), i1 + 1); } - private void indexTifSourceConfig(int num, String configId, String iocIndexName, int i) throws IOException { + private void indexTifSourceConfig(int num, String configId, String indexPattern, String iocActiveIndex, int i) throws IOException { SATIFSourceConfig config = new SATIFSourceConfig( configId, SATIFSourceConfig.NO_VERSION, @@ -96,8 +97,9 @@ private void indexTifSourceConfig(int num, String configId, String iocIndexName, null, null, false, - new DefaultIocStoreConfig(Map.of("ipv4_addr", List.of(iocIndexName))), - List.of("ipv4_addr") + new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.ipv4_addr, indexPattern, iocActiveIndex))), + List.of("ipv4_addr"), + true ); String indexName = SecurityAnalyticsPlugin.JOB_INDEX_NAME; Response response = indexDoc(indexName, configId, config.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString());