From c223d1cecac2936a53fead2c7628e367641dd1ae Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 16 Oct 2024 17:54:47 -0700 Subject: [PATCH] update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 (#1358) Signed-off-by: Subhobrata Dey --- .../DetectorIndexManagementService.java | 13 +++++++++-- .../logtype/LogTypeService.java | 7 +++--- .../services/STIX2IOCFeedStore.java | 11 +++++++++- .../settings/SecurityAnalyticsSettings.java | 4 ++-- .../iocscan/dao/BaseEntityCrudService.java | 7 +++++- .../util/CorrelationIndices.java | 22 +++++++++++++++++-- .../util/CorrelationRuleIndices.java | 11 +++++++++- .../util/CustomLogTypeIndices.java | 6 ++++- .../util/DetectorIndices.java | 10 ++++++++- .../securityanalytics/util/RuleIndices.java | 4 ++++ 10 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index d6cae5304..22b415230 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -558,8 +558,17 @@ private void rolloverIndex( request.getCreateIndexRequest().index(pattern) .mapping(map) .settings(isCorrelation? - Settings.builder().put("index.hidden", true).put("index.correlation", true).build(): - Settings.builder().put("index.hidden", true).build() + Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(): + Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build() ); request.addMaxIndexDocsCondition(docsCondition); request.addMaxIndexAgeCondition(ageCondition); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 56f215a11..9efe57402 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -60,10 +60,10 @@ import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import static org.opensearch.action.support.ActiveShardCount.ALL; import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA; - +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; /** * @@ -456,7 +456,8 @@ public void ensureConfigIndexIsInitialized(ActionListener listener) { isConfigIndexInitialized = false; Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 0eddc8f88..0d545f5be 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -19,6 +19,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.Streams; @@ -49,6 +50,9 @@ import java.util.Map; import java.util.UUID; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; @@ -234,7 +238,12 @@ private void initFeedIndex(String feedIndexName, ActionListener { log.info("Created system index {}", feedIndexName); diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 57b6c5023..53a341234 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -10,10 +10,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.opensearch.index.IndexSettings.MAX_TERMS_COUNT_SETTING; - public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; + public static final int minSystemIndexReplicas = 1; + public static final int maxSystemIndexReplicas = 20; public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java index 62eee1a57..dc5daf891 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -14,6 +14,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; @@ -31,6 +32,8 @@ import java.util.ArrayList; import java.util.List; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; /** @@ -247,7 +250,9 @@ public void createIndexIfNotExists(final ActionListener listener) { public abstract String getEntityName(); protected Settings.Builder getIndexSettings() { - return Settings.builder().put("index.hidden", true); + return Settings.builder().put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas); } public abstract String getEntityAliasName(); diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java index 375342d09..36fd5e37d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -26,6 +27,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationIndices { private static final Logger log = LogManager.getLogger(CorrelationIndices.class); @@ -55,9 +59,15 @@ public static String correlationMappings() throws IOException { public void initCorrelationIndex(ActionListener actionListener) throws IOException { if (!correlationIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_HISTORY_INDEX_PATTERN) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); indexRequest.alias(new Alias(CORRELATION_HISTORY_WRITE_INDEX)); client.admin().indices().create(indexRequest, actionListener); } else { @@ -67,9 +77,15 @@ public void initCorrelationIndex(ActionListener actionListe public void initCorrelationMetadataIndex(ActionListener actionListener) throws IOException { if (!correlationMetadataIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_METADATA_INDEX) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } else { actionListener.onResponse(new CreateIndexResponse(true, true, CORRELATION_METADATA_INDEX)); @@ -136,6 +152,8 @@ public static String correlationAlertIndexMappings() throws IOException { public void initCorrelationAlertIndex(ActionListener actionListener) throws IOException { Settings correlationAlertSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX) .mapping(correlationAlertIndexMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java index d131e47b4..27f6475f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -23,6 +24,9 @@ import java.util.Objects; import org.opensearch.securityanalytics.model.CorrelationRule; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationRuleIndices { private static final Logger log = LogManager.getLogger(CorrelationRuleIndices.class); @@ -45,9 +49,14 @@ public static String correlationRuleIndexMappings() throws IOException { public void initCorrelationRuleIndex(ActionListener actionListener) throws IOException { if (!correlationRuleIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CorrelationRule.CORRELATION_RULE_INDEX).mapping( correlationRuleIndexMappings() - ).settings(Settings.builder().put("index.hidden", true).build()); + ).settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java index 2a065dc57..2769fcee5 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class CustomLogTypeIndices { @@ -42,9 +44,11 @@ public static String customLogTypeMappings() throws IOException { public void initCustomLogTypeIndex(ActionListener actionListener) throws IOException { if (!customLogTypeIndexExists()) { + // Security Analytics log types index is small. 1 primary shard is enough Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX) .mapping(customLogTypeMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java index d6a81e134..83eb058e0 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java @@ -23,6 +23,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class DetectorIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -45,9 +48,14 @@ public static String detectorMappings() throws IOException { public void initDetectorIndex(ActionListener actionListener) throws IOException { if (!detectorIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(Detector.DETECTORS_INDEX) .mapping(detectorMappings()) - .settings(Settings.builder().put("index.hidden", true).build()); + .settings(indexSettings); client.indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 17fe5d802..bb14cacd8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -58,6 +58,8 @@ import java.util.stream.Stream; import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class RuleIndices { @@ -86,6 +88,8 @@ public void initRuleIndex(ActionListener actionListener, bo if (!ruleIndexExists(isPrepackaged)) { Settings indexSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(getRuleIndex(isPrepackaged)) .mapping(ruleMappings())