From dc0e748a880085209852b499aa66ccd5b17430f0 Mon Sep 17 00:00:00 2001 From: naemono Date: Mon, 22 Jan 2018 08:38:30 -0600 Subject: [PATCH 1/3] Pep Fixes Adding timeouts to all requests per http://docs.python-requests.org/en/master/user/quickstart/#timeouts Threading the filtering of data to speed things up with many DCs, and large amounts of data Adding output for exceptions, such as wrong user/pass, which are difficult to debug without Allowing requests timeout to be configurable --- README.md | 12 +++-- gridcheck.py | 1 + gridconfig.py | 2 +- griddata.py | 123 ++++++++++++++++++++++++++++++----------------- requirements.txt | 9 ++-- sensugrid.py | 41 +++++++++++----- 6 files changed, 122 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index b669c6e..26a11b9 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,12 @@ Overview (DCs) # faq -#### how can I filter by more than 1 value? +#### how can I filter by more than 1 value? -Amend the URL and add all the filters together as a comma-separated list, e.g.: +Amend the URL and add all the filters together as a comma-separated list, e.g.: http://localhost:5000/filtered/aaa,bbb,ccc,ddd -#### what do the filters filter by ? +#### what do the filters filter by ? They filter based on the hosts' subscriptions, except in the Events view where they filter on all properties of the check and the host. @@ -85,7 +85,7 @@ Add via pip install or via your package management ``` useradd -r sensu-grid - + ``` ## run as a service @@ -121,10 +121,12 @@ dcs: port: 4567 user: apiuser password: apipassword - + app: refresh: 60 bg_color: #333333 + # This is a python requests layer timeout, as by default, it does not timeout + requests_timeout: 10 ``` ## run locally / manually diff --git a/gridcheck.py b/gridcheck.py index 2c82762..98573a2 100644 --- a/gridcheck.py +++ b/gridcheck.py @@ -1,6 +1,7 @@ import requests import re + def check_connection(dc): url = 'http://{0}:{1}/info'.format(dc['url'], dc['port']) try: diff --git a/gridconfig.py b/gridconfig.py index affa89b..175bcf1 100644 --- a/gridconfig.py +++ b/gridconfig.py @@ -1,6 +1,7 @@ import os import yaml + class Config(object): DEBUG = False TESTING = False @@ -25,4 +26,3 @@ class ProdConfig(Config): class TestingConfig(Config): TESTING = True DEBUG = True - diff --git a/griddata.py b/griddata.py index 531e3aa..5deba3a 100644 --- a/griddata.py +++ b/griddata.py @@ -1,95 +1,123 @@ -from gridcheck import * +import requests +import six +from functools import partial +from multiprocessing.dummy import Pool as ThreadPool -def get_filter_data(dcs): - filter_data = [] - data = None +from gridcheck import check_stash + + +def _filter_data(timeout, dc): + filter_data = list() r = None + data = None + url = 'http://{0}:{1}/clients'.format(dc['url'], dc['port']) + try: + if 'user' and 'password' in dc: + r = requests.get(url, auth=(dc['user'], dc['password']), timeout=timeout) + else: + r = requests.get(url, timeout=timeout) + r.raise_for_status() + except Exception as ex: + print("Got exception while filtering on clients: {}".format(str(ex))) + pass + finally: + if r: + data = r.json() + r.close() + else: + print("no reponse") + + if data: + for i in data: + for s in i['subscriptions']: + if s not in filter_data: + filter_data.append(s) + else: + print("No response data") + return filter_data - for dc in dcs: - url = 'http://{0}:{1}/clients'.format(dc['url'], dc['port']) - try: - if 'user' and 'password' in dc: - r = requests.get(url, auth=(dc['user'], dc['password'])) - else: - r = requests.get(url) - except Exception: - pass - finally: - if r: - data = r.json() - r.close() - - if data: - for i in data: - for s in i['subscriptions']: - if s not in filter_data: - filter_data.append(s) - - if filter_data: - assert type(filter_data) == list - return filter_data + +def get_filter_data(dcs, timeout): + aggregated = list() + pool = ThreadPool(len(dcs)) + func = partial(_filter_data, timeout) + aggregated = pool.map(func, dcs) + if aggregated: + assert type(aggregated) == list + return aggregated return [] -def get_data(dc): +def get_data(dc, timeout): url = 'http://{0}:{1}/results'.format(dc['url'], dc['port']) data = None r = None try: if 'user' and 'password' in dc: - r = requests.get(url, auth=(dc['user'], dc['password'])) + r = requests.get(url, auth=(dc['user'], dc['password']), timeout=timeout) else: - r = requests.get(url) - - except Exception: + r = requests.get(url, timeout=timeout) + r.raise_for_status() + except Exception as ex: + print("Got exception while retrieving data for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: data = r.json() r.close() + else: + print("no reponse") return data -def get_clients(dc): +def get_clients(dc, timeout): url = 'http://{0}:{1}/clients'.format(dc['url'], dc['port']) data = None r = None try: if 'user' and 'password' in dc: - r = requests.get(url, auth=(dc['user'], dc['password'])) + r = requests.get(url, auth=(dc['user'], dc['password']), timeout=timeout) + r.raise_for_status() data = r.json() else: - r = requests.get(url) + r = requests.get(url, timeout=timeout) data = r.json() - except Exception: + except Exception as ex: + print("Got exception while retrieving clients for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: r.close() + else: + print("no reponse") return data -def get_stashes(dc): +def get_stashes(dc, timeout): url = 'http://{0}:{1}/silenced'.format(dc['url'], dc['port']) data = None r = None try: if 'user' and 'password' in dc: - r = requests.get(url, auth=(dc['user'], dc['password'])) + r = requests.get(url, auth=(dc['user'], dc['password']), timeout=timeout) + r.raise_for_status() data = r.json() else: - r = requests.get(url) + r = requests.get(url, timeout=timeout) data = r.json() - except Exception: + except Exception as ex: + print("Got exception while retrieving stashes for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: r.close() + else: + print("no reponse") return data @@ -104,7 +132,7 @@ def filter_object(obj, search): if filter_object(value, search): return True else: - return unicode(search) in unicode(obj) + return six.u(search) in six.u(obj) return False @@ -119,7 +147,7 @@ def filter_event(event): return filter_event -def get_events(dc, filters=[]): +def get_events(dc, timeout, filters=[]): url = 'http://{0}:{1}/events'.format(dc['url'], dc['port']) data = [] @@ -127,11 +155,15 @@ def get_events(dc, filters=[]): try: if 'user' and 'password' in dc: - r = requests.get(url, auth=(dc['user'], dc['password'])) + r = requests.get(url, auth=(dc['user'], dc['password']), timeout=timeout) + r.raise_for_status() data = r.json() else: - r = requests.get(url) + r = requests.get(url, timeout=timeout) data = r.json() + except Exception as ex: + print("Got exception while retrieving events for dc: {} ex: {}".format(dc, str(ex))) + pass finally: if r: r.close() @@ -186,7 +218,8 @@ def agg_data(dc, data, stashes, client_data=None, filters=None): if i['check']['name'] == "keepalive" and i['check']['status'] == 2: if not check_stash(stashes, i['client'], i['check']['name']): - # we cannot currently apply filters as keepalive checks do not have subscribers/subscriptions + # we cannot currently apply filters as keepalive checks do + # not have subscribers/subscriptions down += 1 else: ack += 1 diff --git a/requirements.txt b/requirements.txt index 28a3f55..0dafcce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -Flask -PyYAML +Flask==0.12.2 +PyYAML==3.12 argparse -requests -gunicorn \ No newline at end of file +requests==2.18.4 +gunicorn==19.7.1 +six==1.11.0 diff --git a/sensugrid.py b/sensugrid.py index 0cf148c..66f764b 100644 --- a/sensugrid.py +++ b/sensugrid.py @@ -9,8 +9,17 @@ from flask import abort from reverseproxied import ReverseProxied -from griddata import * -from gridconfig import * +from gridcheck import check_connection +from griddata import ( + agg_data, + get_data, + agg_host_data, + get_stashes, + get_filter_data, + get_clients, + get_events +) +from gridconfig import DevConfig from multiprocessing.dummy import Pool as ThreadPool # https://stackoverflow.com/questions/2846653/how-to-use-threading-in-python @@ -24,10 +33,16 @@ app.config.from_object(myconfig) dcs = app.config['DCS'] appcfg = app.config['APPCFG'] +timeout = appcfg.get('requests_timeout', 10) + + +# Python3 doesn't have cmp +def _cmp(x, y): + return (x > y) - (x < y) def get_agg_data(dc): - r = agg_data(dc, get_data(dc), get_stashes(dc)) + r = agg_data(dc, get_data(dc, timeout), get_stashes(dc, timeout)) return r @@ -36,7 +51,7 @@ def root(): aggregated = list() pool = ThreadPool(len(dcs)) aggregated = pool.map(get_agg_data, dcs) - return render_template('data.html', dcs=dcs, data=aggregated, filter_data=get_filter_data(dcs), appcfg=appcfg) + return render_template('data.html', dcs=dcs, data=aggregated, filter_data=get_filter_data(dcs, timeout), appcfg=appcfg) @app.route('/filtered/', methods=['GET']) @@ -44,9 +59,10 @@ def filtered(filters): aggregated = [] for dc in dcs: if check_connection(dc): - aggregated.append(agg_data(dc, get_data(dc), get_stashes(dc), get_clients(dc), filters)) + aggregated.append(agg_data(dc, get_data(dc, timeout), get_stashes( + dc, timeout), get_clients(dc, timeout), filters)) - return render_template('data.html', dcs=dcs, data=aggregated, filter_data=get_filter_data(dcs), appcfg=appcfg) + return render_template('data.html', dcs=dcs, data=aggregated, filter_data=get_filter_data(dcs, timeout), appcfg=appcfg) @app.route('/show/', methods=['GET']) @@ -62,12 +78,13 @@ def showgrid(d, filters=None): else: clients = None - data_detail = agg_host_data(get_data(dc), get_stashes(dc), clients, filters) + data_detail = agg_host_data(get_data(dc, timeout), + get_stashes(dc, timeout), clients, filters) if data_detail: break else: abort(404) - return render_template('detail.html', dc=dc, data=data_detail, filter_data=get_filter_data(dcs), appcfg=appcfg) + return render_template('detail.html', dc=dc, data=data_detail, filter_data=get_filter_data(dcs, timeout), appcfg=appcfg) @app.route('/events/') @@ -82,15 +99,16 @@ def events(d, filters=''): if dc['name'] == d: dc_found = True if check_connection(dc): - results += get_events(dc, filters.split(',')) + results += get_events(dc, timeout, filters.split(',')) break if dc_found is False: abort(404) - results = sorted(results, lambda x, y: cmp(x['check']['status'], y['check']['status']), reverse=True) + results = sorted(results, lambda x, y: _cmp( + x['check']['status'], y['check']['status']), reverse=True) - return render_template('events.html', dc=dc, data=results, filter_data=get_filter_data(dcs), appcfg=appcfg) + return render_template('events.html', dc=dc, data=results, filter_data=get_filter_data(dcs, timeout), appcfg=appcfg) @app.route('/healthcheck', methods=['GET']) @@ -147,6 +165,7 @@ def icon_for_event(event): return 'question-circle' + if __name__ == '__main__': app.run(host='0.0.0.0', From b646fe558015eba2e964babdbecd4a1dbb4a2211 Mon Sep 17 00:00:00 2001 From: naemono Date: Tue, 23 Jan 2018 08:27:47 -0600 Subject: [PATCH 2/3] Using the logging module instead of print --- griddata.py | 34 +++++++++++++++++++++++---------- sensugrid.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/griddata.py b/griddata.py index 5deba3a..c282f82 100644 --- a/griddata.py +++ b/griddata.py @@ -1,3 +1,4 @@ +import logging import requests import six @@ -7,10 +8,14 @@ from gridcheck import check_stash +LOGGER = logging.getLogger(__name__) + + def _filter_data(timeout, dc): filter_data = list() r = None data = None + LOGGER.debug("Retrieving filters for datacenter: {}".format(dc['name'])) url = 'http://{0}:{1}/clients'.format(dc['url'], dc['port']) try: if 'user' and 'password' in dc: @@ -19,14 +24,14 @@ def _filter_data(timeout, dc): r = requests.get(url, timeout=timeout) r.raise_for_status() except Exception as ex: - print("Got exception while filtering on clients: {}".format(str(ex))) + LOGGER.error("Got exception while filtering on clients: {}".format(str(ex))) pass finally: if r: data = r.json() r.close() else: - print("no reponse") + LOGGER.error("no reponse") if data: for i in data: @@ -34,7 +39,8 @@ def _filter_data(timeout, dc): if s not in filter_data: filter_data.append(s) else: - print("No response data") + LOGGER.error("No response data") + LOGGER.debug("Filter Retrieval for datacenter {} complete".format(dc['name'])) return filter_data @@ -51,6 +57,7 @@ def get_filter_data(dcs, timeout): def get_data(dc, timeout): + LOGGER.debug("Retrieving data for datacenter: {}".format(dc['name'])) url = 'http://{0}:{1}/results'.format(dc['url'], dc['port']) data = None r = None @@ -61,19 +68,21 @@ def get_data(dc, timeout): r = requests.get(url, timeout=timeout) r.raise_for_status() except Exception as ex: - print("Got exception while retrieving data for dc: {} ex: {}".format(dc, str(ex))) + LOGGER.error("Got exception while retrieving data for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: data = r.json() r.close() else: - print("no reponse") + LOGGER.error("no reponse") + LOGGER.debug("Data Retrieval for datacenter {} complete".format(dc['name'])) return data def get_clients(dc, timeout): + LOGGER.debug("Retrieving clients for datacenter: {}".format(dc['name'])) url = 'http://{0}:{1}/clients'.format(dc['url'], dc['port']) data = None r = None @@ -87,18 +96,20 @@ def get_clients(dc, timeout): r = requests.get(url, timeout=timeout) data = r.json() except Exception as ex: - print("Got exception while retrieving clients for dc: {} ex: {}".format(dc, str(ex))) + LOGGER.error("Got exception while retrieving clients for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: r.close() else: - print("no reponse") + LOGGER.error("no reponse") + LOGGER.debug("Client Retrieval for datacenter {} complete".format(dc['name'])) return data def get_stashes(dc, timeout): + LOGGER.debug("Retrieving stashes for datacenter: {}".format(dc['name'])) url = 'http://{0}:{1}/silenced'.format(dc['url'], dc['port']) data = None r = None @@ -111,14 +122,15 @@ def get_stashes(dc, timeout): r = requests.get(url, timeout=timeout) data = r.json() except Exception as ex: - print("Got exception while retrieving stashes for dc: {} ex: {}".format(dc, str(ex))) + LOGGER.error("Got exception while retrieving stashes for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: r.close() else: - print("no reponse") + LOGGER.error("no reponse") + LOGGER.debug("Stash Retrieval for datacenter {} complete".format(dc['name'])) return data @@ -148,6 +160,7 @@ def filter_event(event): def get_events(dc, timeout, filters=[]): + LOGGER.debug("Retrieving events for datacenter: {}".format(dc['name'])) url = 'http://{0}:{1}/events'.format(dc['url'], dc['port']) data = [] @@ -162,12 +175,13 @@ def get_events(dc, timeout, filters=[]): r = requests.get(url, timeout=timeout) data = r.json() except Exception as ex: - print("Got exception while retrieving events for dc: {} ex: {}".format(dc, str(ex))) + LOGGER.error("Got exception while retrieving events for dc: {} ex: {}".format(dc, str(ex))) pass finally: if r: r.close() + LOGGER.debug("Events Retrieval for datacenter {} complete".format(dc['name'])) if len(filters) > 0: return filter(filter_events(filters), data) else: diff --git a/sensugrid.py b/sensugrid.py index 66f764b..3035ea6 100644 --- a/sensugrid.py +++ b/sensugrid.py @@ -25,6 +25,9 @@ # https://stackoverflow.com/questions/2846653/how-to-use-threading-in-python import json +import logging +import logging.config + app = Flask(__name__) app.wsgi_app = ReverseProxied(app.wsgi_app) @@ -34,6 +37,56 @@ dcs = app.config['DCS'] appcfg = app.config['APPCFG'] timeout = appcfg.get('requests_timeout', 10) +log_level = appcfg.get('logging_level', 'INFO').upper() +logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "simple": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": log_level, + "formatter": "simple", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "requests": { + "level": "WARNING", + "handlers": ["console"], + "propagate": False + }, + "sensugrid": { + "level": log_level, + "handlers": ["console"], + "propagate": False + }, + "gridcheck": { + "level": log_level, + "handlers": ["console"], + "propagate": False + }, + "gridconfig": { + "level": log_level, + "handlers": ["console"], + "propagate": False + }, + "griddata": { + "level": log_level, + "handlers": ["console"], + "propagate": False + }, + }, + "": { + "level": log_level, + "handlers": ["console"] + } +}) +LOGGER = logging.getLogger(__name__) # Python3 doesn't have cmp From 6e8c9e9bfaa1f0df4ecfbb0ac3ed092c397e9296 Mon Sep 17 00:00:00 2001 From: naemono Date: Tue, 23 Jan 2018 08:28:19 -0600 Subject: [PATCH 3/3] Updating readme to include logging_level --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 26a11b9..4eab6d6 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ app: bg_color: #333333 # This is a python requests layer timeout, as by default, it does not timeout requests_timeout: 10 + logging_level: info ``` ## run locally / manually