Skip to content

Commit

Permalink
Add stats api. (#308)
Browse files Browse the repository at this point in the history
* Add stats api.

The api provides basic information on the plugin usage and performance, including,
- overall status of the plugin health.
- statistics on the feature stores.
- statistics on the internal caches used by the plugin.

* Refactor the getRequest method.

* Fix case-change in IndexMetadata as part of 7.8 upgrade.

* Resolve Review comments

- Refactor to remove the LTRStats dependency from RestLTRStats
- Remove client and clusterService instance variables from LtrQueryParserPlugin.
- Minor Refactoring

* Remove StoreUtils

- move the operations inside the stat suppliers.

* Use aggregates with multi search requests.

* add rest-api spec and test.

* add stats section into advanced functionality doc.

* Addressed review comments.
  • Loading branch information
Rabi Panda authored Aug 28, 2020
1 parent e6a2877 commit 0455051
Show file tree
Hide file tree
Showing 20 changed files with 1,542 additions and 3 deletions.
64 changes: 63 additions & 1 deletion docs/advanced-functionality.rst
Original file line number Diff line number Diff line change
Expand Up @@ -211,4 +211,66 @@ If (and only if) the extra logging Map is accessed, it will be returned as an ad
}
}
]
}
}

=============================
Stats
=============================
The stats API gives the overall plugin status and statistics::

GET /_ltr/_stats

{
"_nodes": {
"total": 1,
"successful": 1,
"failed": 0
},
"cluster_name": "es-cluster",
"stores": {
"_default_": {
"model_count": 10,
"featureset_count": 1,
"feature_count": 0,
"status": "green"
}
},
"status": "green",
"nodes": {
"2QtMvxMvRoOTymAsoQbxhw": {
"cache": {
"feature": {
"eviction_count": 0,
"miss_count": 0,
"hit_count": 0,
"entry_count": 0,
"memory_usage_in_bytes": 0
},
"featureset": {
"eviction_count": 0,
"miss_count": 0,
"hit_count": 0,
"entry_count": 0,
"memory_usage_in_bytes": 0
},
"model": {
"eviction_count": 0,
"miss_count": 0,
"hit_count": 0,
"entry_count": 0,
"memory_usage_in_bytes": 0
}
}
}
}
}

You can also use filters to retrieve a single stat::

GET /_ltr/_stats/{stat}

Also you can limit the information to a single node in the cluster::

GET /_ltr/_stats/nodes/{nodeId}

GET /_ltr/_stats/{stat}/nodes/{nodeId}
29 changes: 27 additions & 2 deletions src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
import com.o19s.es.ltr.action.ClearCachesAction;
import com.o19s.es.ltr.action.CreateModelFromSetAction;
import com.o19s.es.ltr.action.FeatureStoreAction;
import com.o19s.es.ltr.action.LTRStatsAction;
import com.o19s.es.ltr.action.ListStoresAction;
import com.o19s.es.ltr.action.TransportAddFeatureToSetAction;
import com.o19s.es.ltr.action.TransportCacheStatsAction;
import com.o19s.es.ltr.action.TransportClearCachesAction;
import com.o19s.es.ltr.action.TransportCreateModelFromSetAction;
import com.o19s.es.ltr.action.TransportFeatureStoreAction;
import com.o19s.es.ltr.action.TransportLTRStatsAction;
import com.o19s.es.ltr.action.TransportListStoresAction;
import com.o19s.es.ltr.feature.store.StorableElement;
import com.o19s.es.ltr.feature.store.StoredFeature;
Expand All @@ -53,6 +55,13 @@
import com.o19s.es.ltr.rest.RestStoreManager;
import com.o19s.es.ltr.rest.RestAddFeatureToSet;
import com.o19s.es.ltr.rest.RestFeatureStoreCaches;
import com.o19s.es.ltr.rest.RestLTRStats;
import com.o19s.es.ltr.stats.LTRStat;
import com.o19s.es.ltr.stats.LTRStats;
import com.o19s.es.ltr.stats.StatName;
import com.o19s.es.ltr.stats.suppliers.CacheStatsOnNodeSupplier;
import com.o19s.es.ltr.stats.suppliers.PluginHealthStatusSupplier;
import com.o19s.es.ltr.stats.suppliers.StoreStatsSupplier;
import com.o19s.es.ltr.utils.FeatureStoreLoader;
import com.o19s.es.ltr.utils.Suppliers;
import org.apache.lucene.analysis.core.KeywordTokenizer;
Expand Down Expand Up @@ -100,12 +109,15 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;

public class LtrQueryParserPlugin extends Plugin implements SearchPlugin, ScriptPlugin, ActionPlugin, AnalysisPlugin {
private final LtrRankerParserFactory parserFactory;
Expand Down Expand Up @@ -168,6 +180,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
list.add(new RestFeatureStoreCaches());
list.add(new RestCreateModelFromSet());
list.add(new RestAddFeatureToSet());
list.add(new RestLTRStats());
return unmodifiableList(list);
}

Expand All @@ -179,7 +192,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
new ActionHandler<>(ClearCachesAction.INSTANCE, TransportClearCachesAction.class),
new ActionHandler<>(AddFeaturesToSetAction.INSTANCE, TransportAddFeatureToSetAction.class),
new ActionHandler<>(CreateModelFromSetAction.INSTANCE, TransportCreateModelFromSetAction.class),
new ActionHandler<>(ListStoresAction.INSTANCE, TransportListStoresAction.class)));
new ActionHandler<>(ListStoresAction.INSTANCE, TransportListStoresAction.class),
new ActionHandler<>(LTRStatsAction.INSTANCE, TransportLTRStatsAction.class)));
}

@Override
Expand Down Expand Up @@ -240,7 +254,18 @@ public Collection<Object> createComponents(Client client,
}
}
});
return asList(caches, parserFactory);
return asList(caches, parserFactory, getStats(client, clusterService));
}

private LTRStats getStats(Client client, ClusterService clusterService) {
Map<String, LTRStat> stats = new HashMap<>();
stats.put(StatName.CACHE.getName(),
new LTRStat(false, new CacheStatsOnNodeSupplier(caches)));
stats.put(StatName.STORES.getName(),
new LTRStat(true, new StoreStatsSupplier(client, clusterService)));
stats.put(StatName.PLUGIN_STATUS.getName(),
new LTRStat(true, new PluginHealthStatusSupplier(clusterService)));
return new LTRStats(unmodifiableMap(stats));
}

protected FeatureStoreLoader getFeatureStoreLoader() {
Expand Down
180 changes: 180 additions & 0 deletions src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
package com.o19s.es.ltr.action;

import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.FailedNodeException;
import org.elasticsearch.action.support.nodes.BaseNodeRequest;
import org.elasticsearch.action.support.nodes.BaseNodeResponse;
import org.elasticsearch.action.support.nodes.BaseNodesRequest;
import org.elasticsearch.action.support.nodes.BaseNodesResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class LTRStatsAction extends ActionType<LTRStatsAction.LTRStatsNodesResponse> {
public static final String NAME = "cluster:admin/ltr/stats";
public static final LTRStatsAction INSTANCE = new LTRStatsAction();

public LTRStatsAction() {
super(NAME, LTRStatsNodesResponse::new);
}

public static class LTRStatsRequestBuilder
extends ActionRequestBuilder<LTRStatsNodesRequest, LTRStatsNodesResponse> {
private static final String[] nodeIds = null;

public LTRStatsRequestBuilder(ElasticsearchClient client) {
super(client, INSTANCE, new LTRStatsNodesRequest(nodeIds));
}
}

public static class LTRStatsNodeRequest extends BaseNodeRequest {
private final LTRStatsNodesRequest nodesRequest;

public LTRStatsNodeRequest(LTRStatsNodesRequest nodesRequest) {
this.nodesRequest = nodesRequest;
}

public LTRStatsNodeRequest(StreamInput in) throws IOException {
super(in);
nodesRequest = new LTRStatsNodesRequest(in);
}

public LTRStatsNodesRequest getLTRStatsNodesRequest() {
return nodesRequest;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
nodesRequest.writeTo(out);
}
}

public static class LTRStatsNodeResponse extends BaseNodeResponse implements ToXContentFragment {

private final Map<String, Object> statsMap;

LTRStatsNodeResponse(StreamInput in) throws IOException {
super(in);
this.statsMap = in.readMap(StreamInput::readString, StreamInput::readGenericValue);
}

LTRStatsNodeResponse(DiscoveryNode node, Map<String, Object> statsToValues) {
super(node);
this.statsMap = statsToValues;
}

public Map<String, Object> getStatsMap() {
return statsMap;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeMap(statsMap, StreamOutput::writeString, StreamOutput::writeGenericValue);
}

public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
for (Map.Entry<String, Object> stat : statsMap.entrySet()) {
builder.field(stat.getKey(), stat.getValue());
}

return builder;
}
}

public static class LTRStatsNodesRequest extends BaseNodesRequest<LTRStatsNodesRequest> {

public static final String ALL_STATS_KEY = "_all";

private Set<String> statsToBeRetrieved;

public LTRStatsNodesRequest(StreamInput in) throws IOException {
super(in);
statsToBeRetrieved = in.readSet(StreamInput::readString);
}

public LTRStatsNodesRequest(String[] nodeIds) {
super(nodeIds);
statsToBeRetrieved = new HashSet<>();
}

public void setStatsToBeRetrieved(Set<String> statsToBeRetrieved) {
this.statsToBeRetrieved = statsToBeRetrieved;
}

public Set<String> getStatsToBeRetrieved() {
return statsToBeRetrieved;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStringCollection(statsToBeRetrieved);
}
}

public static class LTRStatsNodesResponse extends BaseNodesResponse<LTRStatsNodeResponse> implements ToXContent {
private static final String NODES_KEY = "nodes";
private final Map<String, Object> clusterStats;

public LTRStatsNodesResponse(StreamInput in) throws IOException {
super(in);
clusterStats = in.readMap();
}

public LTRStatsNodesResponse(ClusterName clusterName, List<LTRStatsNodeResponse> nodeResponses,
List<FailedNodeException> failures, Map<String, Object> clusterStats) {
super(clusterName, nodeResponses, failures);
this.clusterStats = clusterStats;
}

public Map<String, Object> getClusterStats() {
return clusterStats;
}

@Override
protected List<LTRStatsNodeResponse> readNodesFrom(StreamInput in) throws IOException {
return in.readList(LTRStatsNodeResponse::new);
}

@Override
protected void writeNodesTo(StreamOutput out, List<LTRStatsNodeResponse> nodeResponses) throws IOException {
out.writeList(nodeResponses);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeMap(clusterStats);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
for (Map.Entry<String, Object> clusterStat : clusterStats.entrySet()) {
builder.field(clusterStat.getKey(), clusterStat.getValue());
}

builder.startObject(NODES_KEY);
for (LTRStatsNodeResponse ltrStats : getNodes()) {
builder.startObject(ltrStats.getNode().getId());
ltrStats.toXContent(builder, params);
builder.endObject();
}
builder.endObject();
return builder;
}
}
}
Loading

0 comments on commit 0455051

Please sign in to comment.