From 01cc9ab6335263d07997cfd050f53704a3977d59 Mon Sep 17 00:00:00 2001 From: Braedon Vickers Date: Sat, 22 Sep 2018 14:03:37 +1200 Subject: [PATCH] Ensure label order is preserved when parsing --- prometheus_es_exporter/__init__.py | 5 ++-- .../cluster_health_parser.py | 17 ++++++++--- .../indices_stats_parser.py | 23 +++++++++----- prometheus_es_exporter/nodes_stats_parser.py | 30 ++++++++++++++----- prometheus_es_exporter/parser.py | 29 +++++++++++++++--- prometheus_es_exporter/utils.py | 7 +++-- tests/utils.py | 2 +- 7 files changed, 85 insertions(+), 28 deletions(-) diff --git a/prometheus_es_exporter/__init__.py b/prometheus_es_exporter/__init__.py index e2e65f2..b723bfd 100644 --- a/prometheus_es_exporter/__init__.py +++ b/prometheus_es_exporter/__init__.py @@ -8,6 +8,7 @@ import sys import time +from collections import OrderedDict from elasticsearch import Elasticsearch from elasticsearch.exceptions import ConnectionTimeout from functools import partial @@ -51,8 +52,8 @@ def group_metrics(metrics): metric_dict = {} for (name_list, label_dict, value) in metrics: metric_name = format_metric_name(name_list) - label_dict = {format_label_key(k): format_label_value(v) - for k, v in label_dict.items()} + label_dict = OrderedDict([(format_label_key(k), format_label_value(v)) + for k, v in label_dict.items()]) if metric_name not in metric_dict: metric_dict[metric_name] = (tuple(label_dict.keys()), {}) diff --git a/prometheus_es_exporter/cluster_health_parser.py b/prometheus_es_exporter/cluster_health_parser.py index 3e7167a..7dd1336 100644 --- a/prometheus_es_exporter/cluster_health_parser.py +++ b/prometheus_es_exporter/cluster_health_parser.py @@ -1,4 +1,5 @@ -from .utils import merge_dicts +from collections import OrderedDict +from .utils import merge_dicts_ordered singular_forms = { 'indices': 'index', @@ -6,7 +7,12 @@ } -def parse_block(block, metric=[], labels={}): +def parse_block(block, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] # Green is 0, so if we add statuses of mutiple blocks together @@ -33,12 +39,15 @@ def parse_block(block, metric=[], labels={}): else: singular_key = key for n_key, n_value in value.items(): - result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts(labels, {singular_key: [n_key]}))) + result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts_ordered(labels, {singular_key: [n_key]}))) return result -def parse_response(response, metric=[]): +def parse_response(response, metric=None): + if metric is None: + metric = [] + result = [] # Create a shallow copy as we are going to modify it diff --git a/prometheus_es_exporter/indices_stats_parser.py b/prometheus_es_exporter/indices_stats_parser.py index 8772b9e..e660a2e 100644 --- a/prometheus_es_exporter/indices_stats_parser.py +++ b/prometheus_es_exporter/indices_stats_parser.py @@ -1,4 +1,5 @@ -from .utils import merge_dicts +from collections import OrderedDict +from .utils import merge_dicts_ordered singular_forms = { 'fields': 'field' @@ -10,7 +11,12 @@ bucket_list_keys = {} -def parse_block(block, metric=[], labels={}): +def parse_block(block, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] for key, value in block.items(): @@ -26,7 +32,7 @@ def parse_block(block, metric=[], labels={}): else: singular_key = key for n_key, n_value in value.items(): - result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts(labels, {singular_key: [n_key]}))) + result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts_ordered(labels, {singular_key: [n_key]}))) else: result.extend(parse_block(value, metric=metric + [key], labels=labels)) elif isinstance(value, list) and key in bucket_list_keys: @@ -34,19 +40,22 @@ def parse_block(block, metric=[], labels={}): for n_value in value: bucket_name = n_value[bucket_name_key] - result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts(labels, {bucket_name_key: [bucket_name]}))) + result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts_ordered(labels, {bucket_name_key: [bucket_name]}))) return result -def parse_response(response, parse_indices=False, metric=[]): +def parse_response(response, parse_indices=False, metric=None): + if metric is None: + metric = [] + result = [] if '_shards' not in response or not response['_shards']['failed']: if parse_indices: for key, value in response['indices'].items(): - result.extend(parse_block(value, metric=metric, labels={'index': [key]})) + result.extend(parse_block(value, metric=metric, labels=OrderedDict({'index': [key]}))) else: - result.extend(parse_block(response['_all'], metric=metric, labels={'index': ['_all']})) + result.extend(parse_block(response['_all'], metric=metric, labels=OrderedDict({'index': ['_all']}))) return result diff --git a/prometheus_es_exporter/nodes_stats_parser.py b/prometheus_es_exporter/nodes_stats_parser.py index ce80395..9f8d4f4 100644 --- a/prometheus_es_exporter/nodes_stats_parser.py +++ b/prometheus_es_exporter/nodes_stats_parser.py @@ -1,4 +1,5 @@ -from .utils import merge_dicts +from collections import OrderedDict +from .utils import merge_dicts_ordered singular_forms = { 'pools': 'pool', @@ -20,7 +21,12 @@ } -def parse_block(block, metric=[], labels={}): +def parse_block(block, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] for key, value in block.items(): @@ -36,7 +42,7 @@ def parse_block(block, metric=[], labels={}): else: singular_key = key for n_key, n_value in value.items(): - result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts(labels, {singular_key: [n_key]}))) + result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts_ordered(labels, {singular_key: [n_key]}))) else: result.extend(parse_block(value, metric=metric + [key], labels=labels)) elif isinstance(value, list) and key in bucket_list_keys: @@ -44,22 +50,30 @@ def parse_block(block, metric=[], labels={}): for n_value in value: bucket_name = n_value[bucket_name_key] - result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts(labels, {bucket_name_key: [bucket_name]}))) + result.extend(parse_block(n_value, metric=metric + [key], labels=merge_dicts_ordered(labels, {bucket_name_key: [bucket_name]}))) return result -def parse_node(node, metric=[], labels={}): - labels = merge_dicts(labels, node_name=[node['name']]) +def parse_node(node, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + + labels = merge_dicts_ordered(labels, node_name=[node['name']]) return parse_block(node, metric=metric, labels=labels) -def parse_response(response, metric=[]): +def parse_response(response, metric=None): + if metric is None: + metric = [] + result = [] if '_nodes' not in response or not response['_nodes']['failed']: for key, value in response['nodes'].items(): - result.extend(parse_node(value, metric=metric, labels={'node_id': [key]})) + result.extend(parse_node(value, metric=metric, labels=OrderedDict({'node_id': [key]}))) return result diff --git a/prometheus_es_exporter/parser.py b/prometheus_es_exporter/parser.py index 530b6c5..9ec59fe 100644 --- a/prometheus_es_exporter/parser.py +++ b/prometheus_es_exporter/parser.py @@ -1,4 +1,12 @@ -def parse_buckets(agg_key, buckets, metric=[], labels={}): +from collections import OrderedDict + + +def parse_buckets(agg_key, buckets, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] for index, bucket in enumerate(buckets): @@ -23,7 +31,12 @@ def parse_buckets(agg_key, buckets, metric=[], labels={}): return result -def parse_buckets_fixed(agg_key, buckets, metric=[], labels={}): +def parse_buckets_fixed(agg_key, buckets, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] for bucket_key, bucket in buckets.items(): @@ -39,7 +52,12 @@ def parse_buckets_fixed(agg_key, buckets, metric=[], labels={}): return result -def parse_agg(agg_key, agg, metric=[], labels={}): +def parse_agg(agg_key, agg, metric=None, labels=None): + if metric is None: + metric = [] + if labels is None: + labels = OrderedDict() + result = [] for key, value in agg.items(): @@ -55,7 +73,10 @@ def parse_agg(agg_key, agg, metric=[], labels={}): return result -def parse_response(response, metric=[]): +def parse_response(response, metric=None): + if metric is None: + metric = [] + result = [] if not response['timed_out']: diff --git a/prometheus_es_exporter/utils.py b/prometheus_es_exporter/utils.py index 3067d83..06b55cd 100644 --- a/prometheus_es_exporter/utils.py +++ b/prometheus_es_exporter/utils.py @@ -1,4 +1,7 @@ -def merge_dicts(*dict_args, **extra_entries): +from collections import OrderedDict + + +def merge_dicts_ordered(*dict_args, **extra_entries): """ Given an arbitrary number of dictionaries, merge them into a single new dictionary. Later dictionaries take precedence if @@ -7,7 +10,7 @@ def merge_dicts(*dict_args, **extra_entries): Extra entries can also be provided via kwargs. These entries have the highest precedence. """ - res = {} + res = OrderedDict() for d in dict_args + (extra_entries,): res.update(d) diff --git a/tests/utils.py b/tests/utils.py index af14302..c81177f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,7 +25,7 @@ def format_metrics(metric_name, label_keys, value_dict): # Converts the parse_response() result into a psuedo-prometheus format # that is useful for comparing results in tests. -# Uses the 'grop_metrics()' function used by the exporter, so effectively +# Uses the 'group_metrics()' function used by the exporter, so effectively # tests that function. def convert_result(result): metric_dict = group_metrics(result)