Skip to content

Commit

Permalink
Include file cache stats in node stats response (#6333)
Browse files Browse the repository at this point in the history
This change exposes the file cache stats as part the node stats API response.

Signed-off-by: Rabi Panda <[email protected]>
adnapibar authored Feb 23, 2023
1 parent 8c1e19c commit 0f3b870
Showing 15 changed files with 382 additions and 13 deletions.
Original file line number Diff line number Diff line change
@@ -7,6 +7,8 @@
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.hamcrest.MatcherAssert;
import org.opensearch.action.admin.cluster.node.stats.NodeStats;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -29,6 +31,7 @@
import org.opensearch.index.Index;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.monitor.fs.FsInfo;
import org.opensearch.repositories.fs.FsRepository;

@@ -413,4 +416,49 @@ private void assertIndexDirectoryDoesNotExist(String... indexNames) {
}
}
}

public void testFileCacheStats() throws Exception {
final String snapshotName = "test-snap";
final String repoName = "test-repo";
final String indexName1 = "test-idx-1";
final Client client = client();
final int numNodes = 2;

internalCluster().ensureAtLeastNumSearchNodes(numNodes);
createIndexWithDocsAndEnsureGreen(1, 100, indexName1);

createRepositoryWithSettings(null, repoName);
takeSnapshot(client, snapshotName, repoName, indexName1);
deleteIndicesAndEnsureGreen(client, indexName1);
assertAllNodesFileCacheEmpty();

restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
assertNodesFileCacheNonEmpty(numNodes);
}

private void assertAllNodesFileCacheEmpty() {
NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
for (NodeStats stats : response.getNodes()) {
FileCacheStats fcstats = stats.getFileCacheStats();
assertNotNull(fcstats);
assertTrue(isFileCacheEmpty(fcstats));
}
}

private void assertNodesFileCacheNonEmpty(int numNodes) {
NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
int nonEmptyFileCacheNodes = 0;
for (NodeStats stats : response.getNodes()) {
FileCacheStats fcstats = stats.getFileCacheStats();
assertNotNull(fcstats);
if (!isFileCacheEmpty(fcstats)) {
nonEmptyFileCacheNodes++;
}
}
assertEquals(numNodes, nonEmptyFileCacheNodes);
}

private boolean isFileCacheEmpty(FileCacheStats stats) {
return stats.getUsed().getBytes() == 0L && stats.getActive().getBytes() == 0L;
}
}
Original file line number Diff line number Diff line change
@@ -41,12 +41,14 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.discovery.DiscoveryStats;
import org.opensearch.http.HttpStats;
import org.opensearch.index.stats.IndexingPressureStats;
import org.opensearch.index.stats.ShardIndexingPressureStats;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.indices.NodeIndicesStats;
import org.opensearch.indices.breaker.AllCircuitBreakerStats;
import org.opensearch.ingest.IngestStats;
@@ -130,6 +132,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
@Nullable
private WeightedRoutingStats weightedRoutingStats;

@Nullable
private FileCacheStats fileCacheStats;

public NodeStats(StreamInput in) throws IOException {
super(in);
timestamp = in.readVLong();
@@ -171,6 +176,11 @@ public NodeStats(StreamInput in) throws IOException {
} else {
weightedRoutingStats = null;
}
if (in.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
fileCacheStats = in.readOptionalWriteable(FileCacheStats::new);
} else {
fileCacheStats = null;
}
}

public NodeStats(
@@ -194,7 +204,8 @@ public NodeStats(
@Nullable ShardIndexingPressureStats shardIndexingPressureStats,
@Nullable SearchBackpressureStats searchBackpressureStats,
@Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats,
@Nullable WeightedRoutingStats weightedRoutingStats
@Nullable WeightedRoutingStats weightedRoutingStats,
@Nullable FileCacheStats fileCacheStats
) {
super(node);
this.timestamp = timestamp;
@@ -217,6 +228,7 @@ public NodeStats(
this.searchBackpressureStats = searchBackpressureStats;
this.clusterManagerThrottlingStats = clusterManagerThrottlingStats;
this.weightedRoutingStats = weightedRoutingStats;
this.fileCacheStats = fileCacheStats;
}

public long getTimestamp() {
@@ -340,6 +352,10 @@ public WeightedRoutingStats getWeightedRoutingStats() {
return weightedRoutingStats;
}

public FileCacheStats getFileCacheStats() {
return fileCacheStats;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
@@ -374,6 +390,9 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_2_6_0)) {
out.writeOptionalWriteable(weightedRoutingStats);
}
if (out.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
out.writeOptionalWriteable(fileCacheStats);
}
}

@Override
@@ -455,6 +474,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (getWeightedRoutingStats() != null) {
getWeightedRoutingStats().toXContent(builder, params);
}
if (getFileCacheStats() != null && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
getFileCacheStats().toXContent(builder, params);
}

return builder;
}
Original file line number Diff line number Diff line change
@@ -209,7 +209,8 @@ public enum Metric {
SHARD_INDEXING_PRESSURE("shard_indexing_pressure"),
SEARCH_BACKPRESSURE("search_backpressure"),
CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"),
WEIGHTED_ROUTING_STATS("weighted_routing");
WEIGHTED_ROUTING_STATS("weighted_routing"),
FILE_CACHE_STATS("file_cache");

private String metricName;

Original file line number Diff line number Diff line change
@@ -121,7 +121,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
NodesStatsRequest.Metric.SHARD_INDEXING_PRESSURE.containedIn(metrics),
NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics),
NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics),
NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics)
NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics),
NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics)
);
}

Original file line number Diff line number Diff line change
@@ -165,6 +165,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
false,
false,
false,
false,
false
);
List<ShardStats> shardsStats = new ArrayList<>();
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.store.remote.filecache;

import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;

import java.io.IOException;

/**
* Statistics on file cache
*
* @opensearch.internal
*/
public class FileCacheStats implements Writeable, ToXContentFragment {

private final long timestamp;
private final long active;
private final long total;
private final long used;
private final long evicted;
private final long removed;
private final long replaced;
private final long hits;
private final long miss;

public FileCacheStats(
final long timestamp,
final long active,
final long total,
final long used,
final long evicted,
final long removed,
final long replaced,
final long hits,
final long miss
) {
this.timestamp = timestamp;
this.active = active;
this.total = total;
this.used = used;
this.evicted = evicted;
this.removed = removed;
this.replaced = replaced;
this.hits = hits;
this.miss = miss;
}

public FileCacheStats(final StreamInput in) throws IOException {
this.timestamp = in.readLong();
this.active = in.readLong();
this.total = in.readLong();
this.used = in.readLong();
this.evicted = in.readLong();
this.removed = in.readLong();
this.replaced = in.readLong();
this.hits = in.readLong();
this.miss = in.readLong();
}

public static short calculatePercentage(long used, long max) {
return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
}

@Override
public void writeTo(final StreamOutput out) throws IOException {
out.writeLong(timestamp);
out.writeLong(active);
out.writeLong(total);
out.writeLong(used);
out.writeLong(evicted);
out.writeLong(removed);
out.writeLong(replaced);
out.writeLong(hits);
out.writeLong(miss);
}

public long getTimestamp() {
return timestamp;
}

public ByteSizeValue getTotal() {
return new ByteSizeValue(total);
}

public ByteSizeValue getActive() {
return new ByteSizeValue(active);
}

public short getActivePercent() {
return calculatePercentage(active, used);
}

public ByteSizeValue getUsed() {
return new ByteSizeValue(used);
}

public short getUsedPercent() {
return calculatePercentage(getUsed().getBytes(), total);
}

public ByteSizeValue getEvicted() {
return new ByteSizeValue(evicted);
}

public ByteSizeValue getRemoved() {
return new ByteSizeValue(removed);
}

public long getReplacedCount() {
return replaced;
}

public long getCacheHits() {
return hits;
}

public long getCacheMiss() {
return miss;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.FILE_CACHE);
builder.field(Fields.TIMESTAMP, getTimestamp());
builder.humanReadableField(Fields.ACTIVE_IN_BYTES, Fields.ACTIVE, getActive());
builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
builder.humanReadableField(Fields.EVICTED_IN_BYTES, Fields.EVICTED, getEvicted());
builder.humanReadableField(Fields.REMOVED_IN_BYTES, Fields.REMOVED, getRemoved());
builder.field(Fields.REPLACED_COUNT, getReplacedCount());
builder.field(Fields.ACTIVE_PERCENT, getActivePercent());
builder.field(Fields.USED_PERCENT, getUsedPercent());
builder.field(Fields.CACHE_HITS, getCacheHits());
builder.field(Fields.CACHE_MISS, getCacheMiss());
builder.endObject();
return builder;
}

static final class Fields {
static final String FILE_CACHE = "file_cache";
static final String TIMESTAMP = "timestamp";
static final String ACTIVE = "active";
static final String ACTIVE_IN_BYTES = "active_in_bytes";
static final String USED = "used";
static final String USED_IN_BYTES = "used_in_bytes";
static final String EVICTED = "evicted";
static final String EVICTED_IN_BYTES = "evicted_in_bytes";
static final String REMOVED = "removed";
static final String REMOVED_IN_BYTES = "removed_in_bytes";
static final String REPLACED_COUNT = "replaced_count";
static final String TOTAL = "total";
static final String TOTAL_IN_BYTES = "total_in_bytes";

static final String ACTIVE_PERCENT = "active_percent";
static final String USED_PERCENT = "used_percent";

static final String CACHE_HITS = "cache_hits";
static final String CACHE_MISS = "cache_miss";
}
}
Loading

0 comments on commit 0f3b870

Please sign in to comment.