From 162113c9b88003023c5a60ce92287e637f76d158 Mon Sep 17 00:00:00 2001 From: Ruizhen Date: Fri, 26 Feb 2021 14:35:27 -0800 Subject: [PATCH] Take cache actual size into consideration and only emit modify cache actions if the cache actual usage is above lower bound --- .../actions/ModifyCacheMaxSizeAction.java | 9 ++++ .../framework/api/persist/SQLParsingUtil.java | 19 ++++++++ .../framework/api/summaries/ResourceUtil.java | 6 +++ .../rca/store/ElasticSearchAnalysisGraph.java | 22 +++++----- .../store/collector/NodeConfigCollector.java | 32 +++++++++++++- .../rca/store/rca/cache/CacheUtil.java | 20 +++------ .../rca/util/NodeConfigCacheReaderUtil.java | 13 ++++++ src/main/proto/inter_node_rpc_service.proto | 1 + .../jvm/LevelOneActionBuilderTest.java | 29 +++++++++++++ .../jvm/LevelThreeActionBuilderTest.java | 35 +++++++++++++++ .../jvm/LevelTwoActionBuilderTest.java | 30 +++++++++++++ .../NodeConfigClusterCollectorTest.java | 2 +- .../collector/NodeConfigCollectorTest.java | 43 ++++++++++++++++++- 13 files changed, 235 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyCacheMaxSizeAction.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyCacheMaxSizeAction.java index 29b1540ff..1bb7782d9 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyCacheMaxSizeAction.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/actions/ModifyCacheMaxSizeAction.java @@ -177,6 +177,7 @@ public static final class Builder { private Long currentCacheMaxSizeInBytes; private Long desiredCacheMaxSizeInBytes; + private Long currentCacheActualSizeInByte; private Long heapMaxSizeInBytes; private final long upperBoundInBytes; private final long lowerBoundInBytes; @@ -197,6 +198,8 @@ private Builder( this.currentCacheMaxSizeInBytes = NodeConfigCacheReaderUtil.readCacheMaxSizeInBytes( appContext.getNodeConfigCache(), esNode, cacheType); + this.currentCacheActualSizeInByte = NodeConfigCacheReaderUtil.readCacheActualSizeInBytes( + appContext.getNodeConfigCache(), esNode, cacheType); this.heapMaxSizeInBytes = NodeConfigCacheReaderUtil.readHeapMaxSizeInBytes( appContext.getNodeConfigCache(), esNode); this.desiredCacheMaxSizeInBytes = null; @@ -261,6 +264,12 @@ public ModifyCacheMaxSizeAction build() { // Ensure desired cache max size is within thresholds desiredCacheMaxSizeInBytes = Math.max(Math.min(desiredCacheMaxSizeInBytes, upperBoundInBytes), lowerBoundInBytes); + //ensure that we do not issue action to lower cache size if the actual cache size usage + //is below the target lower bound in each bucket level + if (!this.isIncrease && this.currentCacheActualSizeInByte != null + && this.currentCacheActualSizeInByte <= lowerBoundInBytes) { + this.canUpdate = false; + } return new ModifyCacheMaxSizeAction(esNode, cacheType, appContext, desiredCacheMaxSizeInBytes, currentCacheMaxSizeInBytes, coolOffPeriodInMillis, canUpdate); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/persist/SQLParsingUtil.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/persist/SQLParsingUtil.java index c0c71f8de..17d82672b 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/persist/SQLParsingUtil.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/persist/SQLParsingUtil.java @@ -83,4 +83,23 @@ public static double readDataFromSqlResult(Result result, Field } return ret; } + + /** + * Sums up the SUM field of all tuples in the SQL result + * @param records the record result from SQL query + * @return sum value + */ + public static Double readSumFromSqlResult(final Result records) { + if (records == null) { + LOG.error("sql result is null"); + return Double.NaN; + } + double size = 0; + // since the flow unit data is aggregated by index, summing the size across indices + if (records.size() > 0) { + size = records.stream().mapToDouble( + record -> record.getValue(MetricsDB.SUM, Double.class)).sum(); + } + return size; + } } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/summaries/ResourceUtil.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/summaries/ResourceUtil.java index de94ebf19..cc8a219f1 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/summaries/ResourceUtil.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/summaries/ResourceUtil.java @@ -89,6 +89,9 @@ public class ResourceUtil { public static final Resource FIELD_DATA_CACHE_MAX_SIZE = Resource.newBuilder() .setResourceEnum(ResourceEnum.FIELD_DATA_CACHE) .setMetricEnum(MetricEnum.CACHE_MAX_SIZE).build(); + public static final Resource FIELD_DATA_CACHE_ACTUAL_SIZE = Resource.newBuilder() + .setResourceEnum(ResourceEnum.FIELD_DATA_CACHE) + .setMetricEnum(MetricEnum.CACHE_ACTUAL_SIZE).build(); public static final Resource SHARD_REQUEST_CACHE_EVICTION = Resource.newBuilder() .setResourceEnum(ResourceEnum.SHARD_REQUEST_CACHE) .setMetricEnum(MetricEnum.CACHE_EVICTION).build(); @@ -98,6 +101,9 @@ public class ResourceUtil { public static final Resource SHARD_REQUEST_CACHE_MAX_SIZE = Resource.newBuilder() .setResourceEnum(ResourceEnum.SHARD_REQUEST_CACHE) .setMetricEnum(MetricEnum.CACHE_MAX_SIZE).build(); + public static final Resource SHARD_REQUEST_CACHE_ACTUAL_SIZE = Resource.newBuilder() + .setResourceEnum(ResourceEnum.SHARD_REQUEST_CACHE) + .setMetricEnum(MetricEnum.CACHE_ACTUAL_SIZE).build(); /** * Read the resourceType name from the ResourceType object diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/ElasticSearchAnalysisGraph.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/ElasticSearchAnalysisGraph.java index a788c8f19..a24652397 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/ElasticSearchAnalysisGraph.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/ElasticSearchAnalysisGraph.java @@ -239,20 +239,12 @@ public void construct() { cacheMaxSize.addTag(TAG_LOCUS, LOCUS_DATA_MASTER_NODE); addLeaf(cacheMaxSize); - NodeConfigCollector nodeConfigCollector = new NodeConfigCollector(RCA_PERIOD, queueCapacity, cacheMaxSize, heapMax); - nodeConfigCollector.addTag(TAG_LOCUS, LOCUS_DATA_MASTER_NODE); - nodeConfigCollector.addAllUpstreams(Arrays.asList(queueCapacity, cacheMaxSize, heapMax)); - NodeConfigClusterCollector nodeConfigClusterCollector = new NodeConfigClusterCollector(nodeConfigCollector); - nodeConfigClusterCollector.addTag(TAG_LOCUS, LOCUS_MASTER_NODE); - nodeConfigClusterCollector.addAllUpstreams(Collections.singletonList(nodeConfigCollector)); - nodeConfigClusterCollector.addTag(TAG_AGGREGATE_UPSTREAM, LOCUS_DATA_NODE); - // Field Data Cache RCA Metric fieldDataCacheEvictions = new Cache_FieldData_Eviction(EVALUATION_INTERVAL_SECONDS); fieldDataCacheEvictions.addTag(TAG_LOCUS, LOCUS_DATA_MASTER_NODE); addLeaf(fieldDataCacheEvictions); - Metric fieldDataCacheSizeGroupByOperation = new AggregateMetric(EVALUATION_INTERVAL_SECONDS, + AggregateMetric fieldDataCacheSizeGroupByOperation = new AggregateMetric(EVALUATION_INTERVAL_SECONDS, Cache_FieldData_Size.NAME, AggregateFunction.SUM, MetricsDB.MAX, ShardStatsDerivedDimension.INDEX_NAME.toString()); @@ -278,7 +270,7 @@ public void construct() { shardRequestHits.addTag(TAG_LOCUS, LOCUS_DATA_MASTER_NODE); addLeaf(shardRequestHits); - Metric shardRequestCacheSizeGroupByOperation = new AggregateMetric(EVALUATION_INTERVAL_SECONDS, + AggregateMetric shardRequestCacheSizeGroupByOperation = new AggregateMetric(EVALUATION_INTERVAL_SECONDS, Cache_Request_Size.NAME, AggregateFunction.SUM, MetricsDB.MAX, ShardStatsDerivedDimension.INDEX_NAME.toString()); @@ -298,6 +290,16 @@ public void construct() { shardRequestCacheClusterRca.addAllUpstreams(Collections.singletonList(shardRequestCacheNodeRca)); shardRequestCacheClusterRca.addTag(TAG_AGGREGATE_UPSTREAM, LOCUS_DATA_NODE); + //node config collector + NodeConfigCollector nodeConfigCollector = new NodeConfigCollector(RCA_PERIOD, queueCapacity, cacheMaxSize, heapMax, + fieldDataCacheSizeGroupByOperation, shardRequestCacheSizeGroupByOperation); + nodeConfigCollector.addTag(TAG_LOCUS, LOCUS_DATA_MASTER_NODE); + nodeConfigCollector.addAllUpstreams(Arrays.asList(queueCapacity, cacheMaxSize, heapMax)); + NodeConfigClusterCollector nodeConfigClusterCollector = new NodeConfigClusterCollector(nodeConfigCollector); + nodeConfigClusterCollector.addTag(TAG_LOCUS, LOCUS_MASTER_NODE); + nodeConfigClusterCollector.addAllUpstreams(Collections.singletonList(nodeConfigCollector)); + nodeConfigClusterCollector.addTag(TAG_AGGREGATE_UPSTREAM, LOCUS_DATA_NODE); + // Cache Health Decider CacheHealthDecider cacheHealthDecider = new CacheHealthDecider( EVALUATION_INTERVAL_SECONDS, 12, fieldDataCacheClusterRca, shardRequestCacheClusterRca, highHeapUsageClusterRca); diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/collector/NodeConfigCollector.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/collector/NodeConfigCollector.java index 2ff3afb9e..bdae19806 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/collector/NodeConfigCollector.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/collector/NodeConfigCollector.java @@ -33,6 +33,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.ThreadPool_QueueCapacity; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.persist.SQLParsingUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.metric.AggregateMetric; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cache.CacheUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey; import java.util.HashMap; import org.apache.logging.log4j.LogManager; @@ -48,6 +50,8 @@ public class NodeConfigCollector extends EsConfigNode { private final ThreadPool_QueueCapacity threadPool_queueCapacity; private final Cache_Max_Size cacheMaxSize; private final Heap_Max heapMaxSize; + private final AggregateMetric fieldDataCacheSizeGroupByAggregation; + private final AggregateMetric shardRequestCacheSizeGroupByAggregation; private final int rcaPeriod; private int counter; private final HashMap configResult; @@ -55,10 +59,14 @@ public class NodeConfigCollector extends EsConfigNode { public NodeConfigCollector(int rcaPeriod, ThreadPool_QueueCapacity threadPool_queueCapacity, Cache_Max_Size cacheMaxSize, - Heap_Max heapMaxSize) { + Heap_Max heapMaxSize, + AggregateMetric fieldDataCacheSizeGroupByAggregation, + AggregateMetric shardRequestCacheSizeGroupByAggregation) { this.threadPool_queueCapacity = threadPool_queueCapacity; this.cacheMaxSize = cacheMaxSize; this.heapMaxSize = heapMaxSize; + this.fieldDataCacheSizeGroupByAggregation = fieldDataCacheSizeGroupByAggregation; + this.shardRequestCacheSizeGroupByAggregation = shardRequestCacheSizeGroupByAggregation; this.rcaPeriod = rcaPeriod; this.counter = 0; this.configResult = new HashMap<>(); @@ -84,6 +92,16 @@ private void collectCacheMaxSize(MetricFlowUnit cacheMaxSize) { collectAndPublishMetric(ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE, shardRequestCacheMaxSize); } + private void collectFieldDataCacheActualSize(MetricFlowUnit fieldDataCacheSizeGroupByOperation) { + final double fieldDataCacheActualSize = SQLParsingUtil.readSumFromSqlResult(fieldDataCacheSizeGroupByOperation.getData()); + collectAndPublishMetric(ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE, fieldDataCacheActualSize); + } + + private void collectShardRequestCacheActualSize(MetricFlowUnit shardRequestCacheSizeGroupByOperation) { + final double shardRequestCacheActualSize = SQLParsingUtil.readSumFromSqlResult(shardRequestCacheSizeGroupByOperation.getData()); + collectAndPublishMetric(ResourceUtil.SHARD_REQUEST_CACHE_ACTUAL_SIZE, shardRequestCacheActualSize); + } + private void collectHeapStats(MetricFlowUnit heapMax) { // total maximum heap size final double heapMaxSize = SQLParsingUtil.readDataFromSqlResult(heapMax.getData(), @@ -140,6 +158,18 @@ public NodeConfigFlowUnit operate() { } collectHeapStats(flowUnit); } + for (MetricFlowUnit flowUnit : fieldDataCacheSizeGroupByAggregation.getFlowUnits()) { + if (flowUnit.isEmpty()) { + continue; + } + collectFieldDataCacheActualSize(flowUnit); + } + for (MetricFlowUnit flowUnit : shardRequestCacheSizeGroupByAggregation.getFlowUnits()) { + if (flowUnit.isEmpty()) { + continue; + } + collectShardRequestCacheActualSize(flowUnit); + } if (counter == rcaPeriod) { counter = 0; diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/CacheUtil.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/CacheUtil.java index 07a9ff5db..f90ab39d7 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/CacheUtil.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/cache/CacheUtil.java @@ -20,6 +20,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.MetricFlowUnit; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.persist.SQLParsingUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,27 +34,20 @@ public class CacheUtil { public static final long MB_TO_BYTES = KB_TO_BYTES * 1024; public static final long GB_TO_BYTES = MB_TO_BYTES * 1024; - public static Double getTotalSizeInKB(final Metric cacheSizeGroupByOperation) { - double totalSizeInKB = 0; + public static Double getTotalSizeInKB(final Metric cacheSizeGroupByOperation) throws IllegalArgumentException { + double totalSizeInByte = 0; if (cacheSizeGroupByOperation.getFlowUnits().size() > 0) { // we expect the Metric to have single flow unit since it is consumed locally MetricFlowUnit flowUnit = cacheSizeGroupByOperation.getFlowUnits().get(0); if (flowUnit.isEmpty() || flowUnit.getData() == null) { - return totalSizeInKB; - } - - // since the flow unit data is aggregated by index, summing the size across indices - if (flowUnit.getData().size() > 0) { - Result records = flowUnit.getData(); - double size = records.stream().mapToDouble( - record -> record.getValue(MetricsDB.SUM, Double.class)).sum(); - totalSizeInKB += getSizeInKB(size); + return totalSizeInByte; } + totalSizeInByte = SQLParsingUtil.readSumFromSqlResult(flowUnit.getData()); } - if (!Double.isNaN(totalSizeInKB)) { - return totalSizeInKB; + if (!Double.isNaN(totalSizeInByte)) { + return totalSizeInByte / ((double) KB_TO_BYTES); } else { throw new IllegalArgumentException("invalid value: {} in getTotalSizeInKB" + Float.NaN); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/util/NodeConfigCacheReaderUtil.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/util/NodeConfigCacheReaderUtil.java index fc7a20851..173826b89 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/util/NodeConfigCacheReaderUtil.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/util/NodeConfigCacheReaderUtil.java @@ -55,6 +55,19 @@ public static Long readCacheMaxSizeInBytes( return null; } + public static Long readCacheActualSizeInBytes( + final NodeConfigCache nodeConfigCache, final NodeKey esNode, final ResourceEnum cacheType) { + try { + if (cacheType.equals(ResourceEnum.FIELD_DATA_CACHE)) { + return (long) nodeConfigCache.get(esNode, ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE); + } + return (long) nodeConfigCache.get(esNode, ResourceUtil.SHARD_REQUEST_CACHE_ACTUAL_SIZE); + } catch (final IllegalArgumentException e) { + LOG.error("Exception while reading cache actual size from Node Config Cache", e); + } + return null; + } + public static Long readHeapMaxSizeInBytes( final NodeConfigCache nodeConfigCache, final NodeKey esNode) { try { diff --git a/src/main/proto/inter_node_rpc_service.proto b/src/main/proto/inter_node_rpc_service.proto index 936955b62..45496c106 100644 --- a/src/main/proto/inter_node_rpc_service.proto +++ b/src/main/proto/inter_node_rpc_service.proto @@ -98,6 +98,7 @@ enum MetricEnum { CACHE_EVICTION = 10 [(additional_fields).name = "cache eviction", (additional_fields).description = "cache eviction count"]; CACHE_HIT = 11 [(additional_fields).name = "cache hit", (additional_fields).description = "cache hit count"]; CACHE_MAX_SIZE = 12 [(additional_fields).name = "cache max size", (additional_fields).description = "max cache size in bytes"]; + CACHE_ACTUAL_SIZE = 13 [(additional_fields).name = "cache actual size", (additional_fields).description = "cache actual size in bytes"]; // Heap HEAP_MAX = 16 [(additional_fields).name = "heap max", (additional_fields).description = "max heap size in bytes"]; diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelOneActionBuilderTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelOneActionBuilderTest.java index 2ef3e3452..a26aa3751 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelOneActionBuilderTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelOneActionBuilderTest.java @@ -139,4 +139,33 @@ public void testNoAvailableAction() { List actions = LevelOneActionBuilder.newBuilder(node, testAppContext, rcaConf).build(); Assert.assertEquals(0, actions.size()); } + + @Test + public void testSuppressActionWhenCacheUsageIsLow() { + final double fielddataCacheSizeInPercent = 0.3; + final double shardRequestCacheSizeInPercent = 0.04; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheSizeInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheSizeInPercent)); + final double fielddataCacheUsageInPercent = 0.02; + final double shardRequestCacheUsageInPercent = 0.02; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheUsageInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheUsageInPercent)); + List actions = LevelOneActionBuilder.newBuilder(node, testAppContext, rcaConf).build(); + deciderActionParser.addActions(actions); + + Assert.assertEquals(1, actions.size()); + long expectedCacheSize; + long currentCacheSize; + ModifyCacheMaxSizeAction requestCacheAction = deciderActionParser.readCacheAction(ResourceEnum.SHARD_REQUEST_CACHE); + Assert.assertNotNull(requestCacheAction); + expectedCacheSize = + (long) ((shardRequestCacheSizeInPercent - shardRequestCacheStepSize) * heapMaxSizeInBytes); + currentCacheSize = (long) (shardRequestCacheSizeInPercent * heapMaxSizeInBytes); + Assert.assertEquals(expectedCacheSize, requestCacheAction.getDesiredCacheMaxSizeInBytes(), 10); + Assert.assertEquals(currentCacheSize, requestCacheAction.getCurrentCacheMaxSizeInBytes(), 10); + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelThreeActionBuilderTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelThreeActionBuilderTest.java index 154ba86f9..70e4914d4 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelThreeActionBuilderTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelThreeActionBuilderTest.java @@ -25,6 +25,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.CacheActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.QueueActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.configs.jvm.LevelThreeActionBuilderConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.jvm.old_gen.LevelOneActionBuilder; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.jvm.old_gen.LevelThreeActionBuilder; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.test_utils.DeciderActionParserUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum; @@ -198,4 +199,38 @@ public void testDownSizeActionableResources() { Assert.assertTrue(cacheClearAction.isActionable()); Assert.assertEquals(2, cacheClearAction.impactedNodes().size()); } + + @Test + public void testSuppressActionWhenCacheUsageIsLow() { + final double fielddataCacheSizeInPercent = 0.3; + final double shardRequestCacheSizeInPercent = 0.04; + final int writeQueueSize = QueueActionConfig.DEFAULT_WRITE_QUEUE_LOWER_BOUND; + final int searchQueueSize = 2000; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheSizeInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheSizeInPercent)); + final double fielddataCacheUsageInPercent = 0.02; + final double shardRequestCacheUsageInPercent = 0.02; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheUsageInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheUsageInPercent)); + dummyCache.put(node, ResourceUtil.WRITE_QUEUE_CAPACITY, writeQueueSize); + dummyCache.put(node, ResourceUtil.SEARCH_QUEUE_CAPACITY, searchQueueSize); + List actions = LevelThreeActionBuilder.newBuilder(node, testAppContext, rcaConf).build(); + deciderActionParser.addActions(actions); + + Assert.assertEquals(3, actions.size()); + long expectedCacheSize; + long currentCacheSize; + ModifyCacheMaxSizeAction requestCacheAction = deciderActionParser.readCacheAction(ResourceEnum.SHARD_REQUEST_CACHE); + Assert.assertNotNull(requestCacheAction); + expectedCacheSize = (long) (CacheActionConfig.DEFAULT_SHARD_REQUEST_CACHE_LOWER_BOUND * heapMaxSizeInBytes); + currentCacheSize = (long) (shardRequestCacheSizeInPercent * heapMaxSizeInBytes); + Assert.assertEquals(expectedCacheSize, requestCacheAction.getDesiredCacheMaxSizeInBytes(), 10); + Assert.assertEquals(currentCacheSize, requestCacheAction.getCurrentCacheMaxSizeInBytes(), 10); + ModifyCacheMaxSizeAction fielddatatCacheAction = deciderActionParser.readCacheAction(ResourceEnum.FIELD_DATA_CACHE); + Assert.assertNull(fielddatatCacheAction); + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelTwoActionBuilderTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelTwoActionBuilderTest.java index e71822aae..2a486a9a8 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelTwoActionBuilderTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/LevelTwoActionBuilderTest.java @@ -24,6 +24,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.CacheActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.configs.QueueActionConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.configs.jvm.LevelTwoActionBuilderConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.jvm.old_gen.LevelOneActionBuilder; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.jvm.old_gen.LevelTwoActionBuilder; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.test_utils.DeciderActionParserUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.ResourceEnum; @@ -353,4 +354,33 @@ public void testWriteQueueHasLargerBucketIndex() throws Exception { Assert.assertEquals(expectedQueueSize, writeQueueAction.getDesiredCapacity()); Assert.assertEquals(writeQueueSize, writeQueueAction.getCurrentCapacity()); } + + @Test + public void testSuppressActionWhenCacheUsageIsLow() { + final double fielddataCacheSizeInPercent = 0.3; + final double shardRequestCacheSizeInPercent = 0.04; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheSizeInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_MAX_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheSizeInPercent)); + final double fielddataCacheUsageInPercent = 0.02; + final double shardRequestCacheUsageInPercent = 0.02; + dummyCache.put(node, ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * fielddataCacheUsageInPercent)); + dummyCache.put(node, ResourceUtil.SHARD_REQUEST_CACHE_ACTUAL_SIZE, + (long)(heapMaxSizeInBytes * shardRequestCacheUsageInPercent)); + List actions = LevelTwoActionBuilder.newBuilder(node, testAppContext, rcaConf).build(); + deciderActionParser.addActions(actions); + + Assert.assertEquals(1, actions.size()); + long expectedCacheSize; + long currentCacheSize; + ModifyCacheMaxSizeAction requestCacheAction = deciderActionParser.readCacheAction(ResourceEnum.SHARD_REQUEST_CACHE); + Assert.assertNotNull(requestCacheAction); + expectedCacheSize = + (long) ((shardRequestCacheSizeInPercent - shardRequestCacheStepSize) * heapMaxSizeInBytes); + currentCacheSize = (long) (shardRequestCacheSizeInPercent * heapMaxSizeInBytes); + Assert.assertEquals(expectedCacheSize, requestCacheAction.getDesiredCacheMaxSizeInBytes(), 10); + Assert.assertEquals(currentCacheSize, requestCacheAction.getCurrentCacheMaxSizeInBytes(), 10); + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigClusterCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigClusterCollectorTest.java index 1b7da2fe6..1280aca1b 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigClusterCollectorTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigClusterCollectorTest.java @@ -39,7 +39,7 @@ public class NodeConfigClusterCollectorTest { @Before public void init() { - collector = new NodeConfigCollector(1, null, null, null); + collector = new NodeConfigCollector(1, null, null, null, null, null); clusterCollector = new NodeConfigClusterCollector(collector); observer = new RcaTestHelper<>(); AppContext appContext = new AppContext(); diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigCollectorTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigCollectorTest.java index 98065e6ce..e22c98c3b 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigCollectorTest.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/store/collector/NodeConfigCollectorTest.java @@ -21,17 +21,22 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.GCType; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.GradleTaskForRca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.MetricFlowUnit; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.NodeConfigFlowUnit; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_FieldData_Size; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Max_Size; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Cache_Request_Size; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.Heap_Max; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.metrics.ThreadPool_QueueCapacity; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.flow_units.MetricFlowUnitTestHelper; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.collector.NodeConfigCollector; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.metric.AggregateMetric; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.metric.AggregateMetric.AggregateFunction; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey; import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader.ClusterDetailsEventProcessor; import java.util.Arrays; @@ -49,13 +54,24 @@ public class NodeConfigCollectorTest { private Cache_Max_Size cacheMaxSize; private Heap_Max heapMax; private NodeConfigCollector nodeConfigCollector; + private AggregateMetric fieldDataCacheSizeGroupByAggregation; + private AggregateMetric shardRequestCacheSizeGroupByAggregation; @Before public void init() { threadPool_QueueCapacity = new ThreadPool_QueueCapacity(); cacheMaxSize = new Cache_Max_Size(5); heapMax = new Heap_Max(5); - nodeConfigCollector = new NodeConfigCollector(1, threadPool_QueueCapacity, cacheMaxSize, heapMax); + fieldDataCacheSizeGroupByAggregation = new AggregateMetric(5, + Cache_FieldData_Size.NAME, + AggregateFunction.SUM, + MetricsDB.MAX, ShardStatsDerivedDimension.INDEX_NAME.toString()); + shardRequestCacheSizeGroupByAggregation = new AggregateMetric(5, + Cache_Request_Size.NAME, + AggregateFunction.SUM, + MetricsDB.MAX, ShardStatsDerivedDimension.INDEX_NAME.toString()); + nodeConfigCollector = new NodeConfigCollector(1, threadPool_QueueCapacity, cacheMaxSize, heapMax, + fieldDataCacheSizeGroupByAggregation, shardRequestCacheSizeGroupByAggregation); ClusterDetailsEventProcessor clusterDetailsEventProcessor = new ClusterDetailsEventProcessor(); ClusterDetailsEventProcessor.NodeDetails node1 = @@ -97,6 +113,21 @@ private void mockHeapMaxSizeFlowUnits(int heapMaxSize, int oldGenMaxSize, int ed heapMax.setLocalFlowUnit(flowUnit); } + /** + * generate flowunit and bind the flowunits it generate to metrics + */ + @SuppressWarnings("unchecked") + private void mockFieldDataCacheSizeFlowUnits() { + MetricFlowUnit flowUnit = + MetricFlowUnitTestHelper.createFlowUnit( + Arrays.asList(ShardStatsDerivedDimension.INDEX_NAME.toString(), MetricsDB.SUM), + Arrays.asList("index1", "500"), + Arrays.asList("index2", "1500"), + Arrays.asList("index4", "900") + ); + fieldDataCacheSizeGroupByAggregation.setLocalFlowUnit(flowUnit); + } + @Test public void testQueueCapacityMetricNotExist() { threadPool_QueueCapacity.setLocalFlowUnit(MetricFlowUnit.generic()); @@ -160,4 +191,14 @@ public void testHeapMaxSizeCollection() { Assert.assertEquals(oldGenMaxSize, appContext.getNodeConfigCache().get(nodeKey, ResourceUtil.OLD_GEN_MAX_SIZE), 0.01); Assert.assertEquals(expectedYoungGenMaxSize, appContext.getNodeConfigCache().get(nodeKey, ResourceUtil.YOUNG_GEN_MAX_SIZE), 0.01); } + + @Test + public void testFieldDataCacheActualSizeCollection() { + mockFieldDataCacheSizeFlowUnits(); + NodeConfigFlowUnit flowUnit = nodeConfigCollector.operate(); + Assert.assertFalse(flowUnit.isEmpty()); + Assert.assertTrue(flowUnit.hasConfig(ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE)); + Assert.assertEquals(2900, flowUnit.readConfig(ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE), 0.01); + Assert.assertEquals(2900, appContext.getNodeConfigCache().get(nodeKey, ResourceUtil.FIELD_DATA_CACHE_ACTUAL_SIZE), 0.01); + } }