Skip to content

Commit

Permalink
Support retrieving fielddata info for specific fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@braedon
braedon committed Jun 19, 2017
1 parent 8a75332 commit cacf2a7
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 9 deletions.
18 changes: 15 additions & 3 deletions prometheus_es_exporter/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,14 +134,15 @@ def collect(self):


class IndicesStatsCollector(object):
def __init__(self, es_client, parse_indices, metrics=None):
def __init__(self, es_client, parse_indices, metrics=None, fields=None):
self.es_client = es_client
self.parse_indices = parse_indices
self.metrics = metrics
self.fields = fields

def collect(self):
try:
response = self.es_client.indices.stats(metric=self.metrics)
response = self.es_client.indices.stats(metric=self.metrics, fields=self.fields)

metrics = indices_stats_parser.parse_response(response, self.parse_indices, ['es', 'indices_stats'])
except Exception:
Expand Down Expand Up @@ -225,6 +226,13 @@ def csv_choice_arg_parser(choices, arg):
indices_stats_metrics_parser = partial(csv_choice_arg_parser, INDICES_STATS_METRICS_OPTIONS)


def indices_stats_fields_parser(arg):
if arg == '*':
return arg
else:
return arg.split(',')


def main():
signal.signal(signal.SIGTERM, signal_handler)

Expand Down Expand Up @@ -253,6 +261,8 @@ def main():
help='detail mode for indices stats monitoring. (default: cluster)')
parser.add_argument('--indices-stats-metrics', type=indices_stats_metrics_parser,
help='limit indices stats to specific metrics. Metrics should be separated by commas e.g. indices,fs.')
parser.add_argument('--indices-stats-fields', type=indices_stats_fields_parser,
help='include fielddata info for specific fields. Fields should be separated by commas e.g. indices,fs. Use \'*\' for all.')
parser.add_argument('-j', '--json-logging', action='store_true',
help='turn on json logging.')
parser.add_argument('--log-level', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
Expand Down Expand Up @@ -315,7 +325,9 @@ def main():

if not args.indices_stats_disable:
parse_indices = args.indices_stats_mode == 'indices'
REGISTRY.register(IndicesStatsCollector(es_client, parse_indices, metrics=args.indices_stats_metrics))
REGISTRY.register(IndicesStatsCollector(es_client, parse_indices,
metrics=args.indices_stats_metrics,
fields=args.indices_stats_fields))

logging.info('Starting server...')
start_http_server(port)
Expand Down
8 changes: 6 additions & 2 deletions prometheus_es_exporter/indices_stats_parser.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
singular_forms = {}
singular_forms = {
'fields': 'field'
}
excluded_keys = []
bucket_dict_keys = []
bucket_dict_keys = [
'fields'
]
bucket_list_keys = {}


Expand Down
48 changes: 44 additions & 4 deletions tests/test_indices_stats_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,15 @@ class Test(unittest.TestCase):
},
'fielddata': {
'memory_size_in_bytes': 0,
'evictions': 0
'evictions': 0,
'fields': {
'group1': {
'memory_size_in_bytes': 1024
},
'group2': {
'memory_size_in_bytes': 2048
}
}
},
'completion': {
'size_in_bytes': 0
Expand Down Expand Up @@ -219,7 +227,15 @@ class Test(unittest.TestCase):
},
'fielddata': {
'memory_size_in_bytes': 0,
'evictions': 0
'evictions': 0,
'fields': {
'group1': {
'memory_size_in_bytes': 1024
},
'group2': {
'memory_size_in_bytes': 2048
}
}
},
'completion': {
'size_in_bytes': 0
Expand Down Expand Up @@ -339,7 +355,15 @@ class Test(unittest.TestCase):
},
'fielddata': {
'memory_size_in_bytes': 0,
'evictions': 0
'evictions': 0,
'fields': {
'group1': {
'memory_size_in_bytes': 1024
},
'group2': {
'memory_size_in_bytes': 2048
}
}
},
'completion': {
'size_in_bytes': 0
Expand Down Expand Up @@ -456,7 +480,15 @@ class Test(unittest.TestCase):
},
'fielddata': {
'memory_size_in_bytes': 0,
'evictions': 0
'evictions': 0,
'fields': {
'group1': {
'memory_size_in_bytes': 1024
},
'group2': {
'memory_size_in_bytes': 2048
}
}
},
'completion': {
'size_in_bytes': 0
Expand Down Expand Up @@ -559,6 +591,8 @@ def test_endpoint_cluster(self):
'primaries_query_cache_evictions{index="_all"}': 0,
'primaries_fielddata_memory_size_in_bytes{index="_all"}': 0,
'primaries_fielddata_evictions{index="_all"}': 0,
'primaries_fielddata_fields_memory_size_in_bytes{field="group1",index="_all"}': 1024,
'primaries_fielddata_fields_memory_size_in_bytes{field="group2",index="_all"}': 2048,
'primaries_completion_size_in_bytes{index="_all"}': 0,
'primaries_segments_count{index="_all"}': 3,
'primaries_segments_memory_in_bytes{index="_all"}': 7908,
Expand Down Expand Up @@ -641,6 +675,8 @@ def test_endpoint_cluster(self):
'total_query_cache_evictions{index="_all"}': 0,
'total_fielddata_memory_size_in_bytes{index="_all"}': 0,
'total_fielddata_evictions{index="_all"}': 0,
'total_fielddata_fields_memory_size_in_bytes{field="group1",index="_all"}': 1024,
'total_fielddata_fields_memory_size_in_bytes{field="group2",index="_all"}': 2048,
'total_completion_size_in_bytes{index="_all"}': 0,
'total_segments_count{index="_all"}': 3,
'total_segments_memory_in_bytes{index="_all"}': 7908,
Expand Down Expand Up @@ -730,6 +766,8 @@ def test_endpoint_indices(self):
'primaries_query_cache_evictions{index="foo"}': 0,
'primaries_fielddata_memory_size_in_bytes{index="foo"}': 0,
'primaries_fielddata_evictions{index="foo"}': 0,
'primaries_fielddata_fields_memory_size_in_bytes{field="group1",index="foo"}': 1024,
'primaries_fielddata_fields_memory_size_in_bytes{field="group2",index="foo"}': 2048,
'primaries_completion_size_in_bytes{index="foo"}': 0,
'primaries_segments_count{index="foo"}': 3,
'primaries_segments_memory_in_bytes{index="foo"}': 7908,
Expand Down Expand Up @@ -812,6 +850,8 @@ def test_endpoint_indices(self):
'total_query_cache_evictions{index="foo"}': 0,
'total_fielddata_memory_size_in_bytes{index="foo"}': 0,
'total_fielddata_evictions{index="foo"}': 0,
'total_fielddata_fields_memory_size_in_bytes{field="group1",index="foo"}': 1024,
'total_fielddata_fields_memory_size_in_bytes{field="group2",index="foo"}': 2048,
'total_completion_size_in_bytes{index="foo"}': 0,
'total_segments_count{index="foo"}': 3,
'total_segments_memory_in_bytes{index="foo"}': 7908,
Expand Down

0 comments on commit cacf2a7

Please sign in to comment.