From 20a65f1aab0c831589a2f81ebbacef9dc4d15296 Mon Sep 17 00:00:00 2001 From: Huang ChuanTong Date: Tue, 15 Apr 2014 09:15:35 +0800 Subject: [PATCH 1/7] add argument support: -s , --second seconds of the records save in the memory [default: 20] when insert records more then 2w, the sqlite will slow. so clear it pre 20 second. --- ngxtop/ngxtop.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index afb5bc3..5a9d530 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -18,6 +18,7 @@ -w , --having having clause [default: 1] -o , --order-by order of output for default query [default: count] -n , --limit limit the number of records included in report for top command [default: 10] + -s , --second seconds of the records save in the memory [default: 20] -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output -v, --verbose more verbose output @@ -198,9 +199,10 @@ def parse_log(lines, pattern): # Records and statistic processor # ================================= class SQLProcessor(object): - def __init__(self, report_queries, fields, index_fields=None): + def __init__(self, report_queries, fields, index_fields=None, second=20): self.begin = False self.report_queries = report_queries + self.second = second self.index_fields = index_fields if index_fields is not None else [] self.column_list = ','.join(fields) self.holder_list = ','.join(':%s' % var for var in fields) @@ -232,6 +234,10 @@ def report(self): columns = (d[0] for d in cursor.description) result = tabulate.tabulate(cursor.fetchall(), headers=columns, tablefmt='orgtbl', floatfmt='.3f') output.append('%s\n%s' % (label, result)) + now = time.time() + if now - self.begin >= self.second: + cursor.execute("delete from log;") + self.begin = now return '\n\n'.join(output) def init_db(self): @@ -306,7 +312,8 @@ def build_processor(arguments): for field in fields: processor_fields.extend(field.split(',')) - processor = SQLProcessor(report_queries, processor_fields) + second = int(arguments['--second']) if arguments['--second'] else 20 + processor = SQLProcessor(report_queries, fields, second=second) return processor From d42f7a64ff50f6dc1e46aee4ccd2bee9278e8a5c Mon Sep 17 00:00:00 2001 From: toontong Date: Fri, 16 May 2014 16:04:31 +0800 Subject: [PATCH 2/7] add the email processor. support multiple processor. --- README.rst | 25 +++++- ngxtop/emailprocessor .py | 111 ++++++++++++++++++++++++ ngxtop/ngxtop.py | 161 +++++------------------------------ ngxtop/processor.py | 10 +++ ngxtop/sqlprocessor.py | 172 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 336 insertions(+), 143 deletions(-) create mode 100644 ngxtop/emailprocessor .py create mode 100644 ngxtop/processor.py create mode 100644 ngxtop/sqlprocessor.py diff --git a/README.rst b/README.rst index 2680597..8e9db8c 100644 --- a/README.rst +++ b/README.rst @@ -32,19 +32,28 @@ Usage ngxtop info Options: + -r , --reportor choice in[sql, email] or other Implemented by your self(see the source code sqlprocessor.py). [default: sql] -l , --access-log access log file to parse. - -f , --log-format log format as specify in log_format directive. + -f , --log-format log format as specify in log_format directive. [default: combined] --no-follow ngxtop default behavior is to ignore current lines in log and only watch for new lines as they are written to the access log. Use this flag to tell ngxtop to process the current content of the access log instead. -t , --interval report interval when running in follow mode [default: 2.0] - -g , --group-by group by variable [default: request_path] + -g , --group-by group by variable [default: request_path]. -w , --having having clause [default: 1] -o , --order-by order of output for default query [default: count] -n , --limit limit the number of records included in report for top command [default: 10] + -s , --second seconds of the records save in the memory [default: 20] -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output - + + -e , --email ... email to who. + -S , --smtp smtp service. + -u , --user smtp auth user. + -P , --password smtp auth user's passwod. + -F , --from who send the email. + -T , --subject email Subject-title [default: (hostname)ngxtop-access-log-email-notify]. + -v, --verbose more verbose output -d, --debug print every line and parsed record -h, --help print this help message. @@ -147,3 +156,13 @@ Parse apache log from remote server with `common` format | /xxxxx/ | 14 | 0.000 | 0 | 14 | 0 | 0 | | /xxxxxxxxxx/xxxxxxxx/xxxxx | 13 | 20530.154 | 13 | 0 | 0 | 0 | +Use the `email` for the porcessor-report the `5xx` error real-time(check every 10 second.): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + $ ngxtop --filter '5 == status_type' \ + -e 'user_who_want_recevc_the_email@xxx.com;user2@163.com'\ + -F 'email_send_from' \ + -S 'smtp_server' \ + -u 'smtp_auth_user' \ + -P '****password***' \ + -t 10 \ No newline at end of file diff --git a/ngxtop/emailprocessor .py b/ngxtop/emailprocessor .py new file mode 100644 index 0000000..8597eac --- /dev/null +++ b/ngxtop/emailprocessor .py @@ -0,0 +1,111 @@ +import time +import socket +import pprint +import logging +import smtplib +from datetime import datetime +from email.mime.text import MIMEText + +import tabulate +from processor import BaseProcessor + + +class EMailProcessor(BaseProcessor): + #def __init__(self, report_queries, fields, index_fields=None, second=20): + def __init__(self, arguments): + self.emails_to = arguments['--email'] + self.smtp = arguments['--smtp'] + self.user = arguments['--user'] + self.password = arguments['--password'] + self.sender = arguments['--from'] + self.subject = '[%s]-%s' % (socket.gethostname(), arguments['--subject']) + self.no_follow = arguments['--no-follow'] + self.debug = arguments['--debug'] or arguments['--verbose'] + + fmt = arguments['--log-format'].replace('-', '') + fmt = fmt.replace('[', '') + fmt = fmt.replace(']', '') + fmt = fmt.replace('$', '') + + self.logfmtkeys = [k for k in fmt.split(' ') if k] + self.begin = 0 + self.access_log_buffer = [] + self.summary = {} + self.detail = {} + + def process(self, records): + self.begin = time.time() + for r in records: + self.access_log_buffer.append(r) + try: + status_code_key = '%sxx' % r['status_type'] + self.summary[status_code_key] = 1 + self.summary.setdefault(status_code_key, 0) + self.summary['count'] = 1 + self.summary.setdefault('count', 0) + + path = r['request_path'] + path_info = self.detail.setdefault(path, {}) + path_info['count'] = 1 + path_info.setdefault('count', 0) + path_info[r['status'] ] = 1 + path_info.setdefault(r['status'] , 0) + + except Exception as e: + logging.warning('log-record can not parse.[%s]. Exception[%s]', r, e) + + def _make_report(self, summary, detail, access_log_buffer): + title = '************ host[%s] date[%s] **************' % (socket.gethostname(), datetime.now()) + split = '\n------------------- %s --------------------\n' + lst = [title, split % 'Summary'] + lst.append(pprint.pformat(summary, indent=4)) + lst.append(split % 'Detailed') + lst.append(pprint.pformat(detail, indent=4)) + lst.append(split % 'Access Logs[ Limit 10]') + lst.append(pprint.pformat(access_log_buffer[:10], indent=2)) + return '\n'.join(lst) + + def report(self): + if not self.begin: + logging.warning('process did not begin.') + return '' + + if not self.access_log_buffer: + logging.debug('access-log buffer is empty.') + return 'access-log buffer is empty.' + + summary, detail, access_log_buffer = self.summary, self.detail, self.access_log_buffer + + hr, res = self._send_mail(self._make_report(summary, detail, access_log_buffer)) + + if hr: + self.summary, self.detail, self.access_log_buffer = {}, {}, [] + + msg = '[%s] send report to then email[%s] --> [%s].' % (datetime.now(), self.emails_to, res) + + if self.no_follow: + print(msg) + else: + return msg + + def _send_mail(self, content): + logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]', + self.subject, self.emails_to, self.smtp, self.user) + msg = MIMEText(content, 'plain', _charset='utf-8') + if self.debug: + logging.debug('email content:\n%s', content) + return True, 'just-test, email did not send.' + + msg['Subject'] = self.subject + msg['From'] = self.sender + msg['To'] = self.emails_to + try: + s = smtplib.SMTP() + s.connect(self.smtp) + s.login(self.user, self.password) + s.sendmail(self.sender, self.emails_to, msg.as_string()) + s.close() + logging.info('email was send to[%s]', self.emails_to) + return True, 'success' + except Exception, e: + logging.error('send_mail to[%s] Exception[%s]', + self.emails_to, e) + return False, 'fail' + + diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 5a9d530..b888d11 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -7,6 +7,7 @@ ngxtop [options] query ... Options: + -r , --reportor choice in[sql, email] or other Implemented by your self(see the source code sqlprocessor.py). [default: sql] -l , --access-log access log file to parse. -f , --log-format log format as specify in log_format directive. [default: combined] --no-follow ngxtop default behavior is to ignore current lines in log @@ -14,13 +15,20 @@ Use this flag to tell ngxtop to process the current content of the access log instead. -t , --interval report interval when running in follow mode [default: 2.0] - -g , --group-by group by variable [default: request_path] + -g , --group-by group by variable [default: request_path]. -w , --having having clause [default: 1] -o , --order-by order of output for default query [default: count] -n , --limit limit the number of records included in report for top command [default: 10] -s , --second seconds of the records save in the memory [default: 20] -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output - + + -e , --email ... email to who. + -S , --smtp smtp service. + -u , --user smtp auth user. + -P , --password smtp auth user's passwod. + -F , --from who send the email. + -T , --subject email Subject-title [default: (hostname)ngxtop-access-log-email-notify]. + -v, --verbose more verbose output -d, --debug print every line and parsed record -h, --help print this help message. @@ -30,7 +38,7 @@ -c , --config allow ngxtop to parse nginx config file for log format and location. -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. - + Examples: All examples read nginx config file for access log location and format. If you want to specify the access log file and / or log format, use the -f and -a options. @@ -77,39 +85,8 @@ from config_parser import detect_log_config, detect_config_path, extract_variables, build_pattern from utils import error_exit - - -DEFAULT_QUERIES = [ - ('Summary:', - '''SELECT - count(1) AS count, - avg(bytes_sent) AS avg_bytes_sent, - count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', - count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', - count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' - FROM log - ORDER BY %(--order-by)s DESC - LIMIT %(--limit)s'''), - - ('Detailed:', - '''SELECT - %(--group-by)s, - count(1) AS count, - avg(bytes_sent) AS avg_bytes_sent, - count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', - count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', - count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' - FROM log - GROUP BY %(--group-by)s - HAVING %(--having)s - ORDER BY %(--order-by)s DESC - LIMIT %(--limit)s''') -] - -DEFAULT_FIELDS = set(['status_type', 'bytes_sent']) - +from sqlprocessor import SQLProcessor +from emailprocessor import EMailProcessor # ====================== # generator utilities @@ -194,68 +171,6 @@ def parse_log(lines, pattern): records = add_field('request_path', parse_request_path, records) return records - -# ================================= -# Records and statistic processor -# ================================= -class SQLProcessor(object): - def __init__(self, report_queries, fields, index_fields=None, second=20): - self.begin = False - self.report_queries = report_queries - self.second = second - self.index_fields = index_fields if index_fields is not None else [] - self.column_list = ','.join(fields) - self.holder_list = ','.join(':%s' % var for var in fields) - self.conn = sqlite3.connect(':memory:') - self.init_db() - - def process(self, records): - self.begin = time.time() - insert = 'insert into log (%s) values (%s)' % (self.column_list, self.holder_list) - logging.info('sqlite insert: %s', insert) - with closing(self.conn.cursor()) as cursor: - for r in records: - cursor.execute(insert, r) - - def report(self): - if not self.begin: - return '' - count = self.count() - duration = time.time() - self.begin - status = 'running for %.0f seconds, %d records processed: %.2f req/sec' - output = [status % (duration, count, count / duration)] - with closing(self.conn.cursor()) as cursor: - for query in self.report_queries: - if isinstance(query, tuple): - label, query = query - else: - label = '' - cursor.execute(query) - columns = (d[0] for d in cursor.description) - result = tabulate.tabulate(cursor.fetchall(), headers=columns, tablefmt='orgtbl', floatfmt='.3f') - output.append('%s\n%s' % (label, result)) - now = time.time() - if now - self.begin >= self.second: - cursor.execute("delete from log;") - self.begin = now - return '\n\n'.join(output) - - def init_db(self): - create_table = 'create table log (%s)' % self.column_list - with closing(self.conn.cursor()) as cursor: - logging.info('sqlite init: %s', create_table) - cursor.execute(create_table) - for idx, field in enumerate(self.index_fields): - sql = 'create index log_idx%d on log (%s)' % (idx, field) - logging.info('sqlite init: %s', sql) - cursor.execute(sql) - - def count(self): - with closing(self.conn.cursor()) as cursor: - cursor.execute('select count(1) from log') - return cursor.fetchone()[0] - - # =============== # Log processing # =============== @@ -271,49 +186,16 @@ def process_log(lines, pattern, processor, arguments): records = (r for r in records if eval(filter_exp, {}, r)) processor.process(records) - print(processor.report()) # this will only run when start in --no-follow mode + processor.report() # this will only run when start in --no-follow mode def build_processor(arguments): - fields = arguments[''] - if arguments['print']: - label = ', '.join(fields) + ':' - selections = ', '.join(fields) - query = 'select %s from log group by %s' % (selections, selections) - report_queries = [(label, query)] - elif arguments['top']: - limit = int(arguments['--limit']) - report_queries = [] - for var in fields: - label = 'top %s' % var - query = 'select %s, count(1) as count from log group by %s order by count desc limit %d' % (var, var, limit) - report_queries.append((label, query)) - elif arguments['avg']: - label = 'average %s' % fields - selections = ', '.join('avg(%s)' % var for var in fields) - query = 'select %s from log' % selections - report_queries = [(label, query)] - elif arguments['sum']: - label = 'sum %s' % fields - selections = ', '.join('sum(%s)' % var for var in fields) - query = 'select %s from log' % selections - report_queries = [(label, query)] - elif arguments['query']: - report_queries = arguments[''] - fields = arguments[''] + if arguments['--reportor'] == 'sql': + processor = SQLProcessor(arguments) + elif arguments['--reportor'] == 'email': + processor = EMailProcessor(arguments) else: - report_queries = [(name, query % arguments) for name, query in DEFAULT_QUERIES] - fields = DEFAULT_FIELDS.union(set([arguments['--group-by']])) - - for label, query in report_queries: - logging.info('query for "%s":\n %s', label, query) - - processor_fields = [] - for field in fields: - processor_fields.extend(field.split(',')) - - second = int(arguments['--second']) if arguments['--second'] else 20 - processor = SQLProcessor(report_queries, fields, second=second) + error_exit('no [%s] Processor was define.' % arguments['--reportor']) return processor @@ -379,13 +261,13 @@ def process(arguments): def main(): args = docopt(__doc__, version='xstat 0.1') - + log_level = logging.WARNING if args['--verbose']: log_level = logging.INFO if args['--debug']: log_level = logging.DEBUG - logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s') + logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s:%(filename)s[%(lineno)d] - %(message)s') logging.debug('arguments:\n%s', args) try: @@ -393,6 +275,5 @@ def main(): except KeyboardInterrupt: sys.exit(0) - if __name__ == '__main__': main() diff --git a/ngxtop/processor.py b/ngxtop/processor.py new file mode 100644 index 0000000..b156772 --- /dev/null +++ b/ngxtop/processor.py @@ -0,0 +1,10 @@ + + +class BaseProcessor(object): + def process(self, records): + '''do something with records like''' + for r in records: + print (r) + + def report(self): + return 'NotImplemented' diff --git a/ngxtop/sqlprocessor.py b/ngxtop/sqlprocessor.py new file mode 100644 index 0000000..87f6130 --- /dev/null +++ b/ngxtop/sqlprocessor.py @@ -0,0 +1,172 @@ +''' +ngxtop sqlprocessor - ad-hoc query for nginx access log. + +Usage: + ngxtop -r sql [options] + ngxtop -r sql [options] (print|top|avg|sum) ... + ngxtop -r sql info + ngxtop -r sql [options] query .. + +Options: + -g , --group-by group by variable [default: request_path] + -w , --having having clause [default: 1] + -o , --order-by order of output for default query [default: count] + -n , --limit limit the number of records included in report for top command [default: 10] + -s , --second seconds of the records save in the memory [default: 20] + -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output + +Examples: + All examples read nginx config file for access log location and format. + If you want to specify the access log file and / or log format, use the -f and -a options. + + "top" like view of nginx requests + $ ngxtop + + Top 10 requests with highest total bytes sent + $ ngxtop --order-by 'avg(bytes_sent) * count' + + Top 10 remote address, e.g., who's hitting you the most + $ ngxtop --group-by remote_addr + +''' +import time +import logging +import sqlite3 +from contextlib import closing +import tabulate +from docopt import docopt + +from processor import BaseProcessor + +DEFAULT_QUERIES = [ + ('Summary:', + '''SELECT + count(1) AS count, + avg(bytes_sent) AS avg_bytes_sent, + count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', + count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', + count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + FROM log + ORDER BY %(--order-by)s DESC + LIMIT %(--limit)s'''), + + ('Detailed:', + '''SELECT + %(--group-by)s, + count(1) AS count, + avg(bytes_sent) AS avg_bytes_sent, + count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', + count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', + count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + FROM log + GROUP BY %(--group-by)s + HAVING %(--having)s + ORDER BY %(--order-by)s DESC + LIMIT %(--limit)s''') +] + +DEFAULT_FIELDS = set(['status_type', 'bytes_sent']) + +# ================================= +# Records and statistic processor +# ================================= +class SQLProcessor(BaseProcessor): + def __init__(self, arguments): + + fields = arguments[''] + if arguments['print']: + label = ', '.join(fields) + ':' + selections = ', '.join(fields) + query = 'select %s from log group by %s' % (selections, selections) + report_queries = [(label, query)] + elif arguments['top']: + limit = int(arguments['--limit']) + report_queries = [] + for var in fields: + label = 'top %s' % var + query = 'select %s, count(1) as count from log group by %s order by count desc limit %d' % (var, var, limit) + report_queries.append((label, query)) + elif arguments['avg']: + label = 'average %s' % fields + selections = ', '.join('avg(%s)' % var for var in fields) + query = 'select %s from log' % selections + report_queries = [(label, query)] + elif arguments['sum']: + label = 'sum %s' % fields + selections = ', '.join('sum(%s)' % var for var in fields) + query = 'select %s from log' % selections + report_queries = [(label, query)] + elif arguments['query']: + report_queries = arguments[''] + fields = arguments[''] + else: + report_queries = [(name, query % arguments) for name, query in DEFAULT_QUERIES] + fields = DEFAULT_FIELDS.union(set([arguments['--group-by']])) + + for label, query in report_queries: + logging.info('query for "%s":\n %s', label, query) + + processor_fields = [] + for field in fields: + processor_fields.extend(field.split(',')) + + self.no_follow = arguments['--no-follow'] + self.begin = False + self.report_queries = report_queries + self.second = int(arguments['--second']) if arguments['--second'] else 20 + self.index_fields = [] #index_fields if index_fields is not None else [] + self.column_list = ','.join(fields) + self.holder_list = ','.join(':%s' % var for var in fields) + self.conn = sqlite3.connect(':memory:') + self.init_db() + + def process(self, records): + self.begin = time.time() + insert = 'insert into log (%s) values (%s)' % (self.column_list, self.holder_list) + logging.info('sqlite insert: %s', insert) + with closing(self.conn.cursor()) as cursor: + for r in records: + cursor.execute(insert, r) + + def report(self): + if not self.begin: + return '' + count = self.count() + duration = time.time() - self.begin + status = 'running for %.0f seconds, %d records processed: %.2f req/sec' + output = [status % (duration, count, count / duration)] + with closing(self.conn.cursor()) as cursor: + for query in self.report_queries: + if isinstance(query, tuple): + label, query = query + else: + label = '' + cursor.execute(query) + columns = (d[0] for d in cursor.description) + result = tabulate.tabulate(cursor.fetchall(), headers=columns, tablefmt='orgtbl', floatfmt='.3f') + output.append('%s\n%s' % (label, result)) + now = time.time() + if now - self.begin >= self.second: + cursor.execute("delete from log;") + self.begin = now + report = '\n\n'.join(output) + if self.no_follow: + print(report) + return report + + def init_db(self): + create_table = 'create table log (%s)' % self.column_list + with closing(self.conn.cursor()) as cursor: + logging.info('sqlite init: %s', create_table) + cursor.execute(create_table) + for idx, field in enumerate(self.index_fields): + sql = 'create index log_idx%d on log (%s)' % (idx, field) + logging.info('sqlite init: %s', sql) + cursor.execute(sql) + + def count(self): + with closing(self.conn.cursor()) as cursor: + cursor.execute('select count(1) from log') + return cursor.fetchone()[0] From b5e92a70ea8715085d1b942bb95c30ae25955cfc Mon Sep 17 00:00:00 2001 From: toontong Date: Fri, 16 May 2014 16:14:02 +0800 Subject: [PATCH 3/7] update version from 0.0.2 to 0.0.3 and add auth email. update version from 0.0.2 to 0.0.3 and add auth email. --- README.rst | 16 +++++++++------- setup.py | 8 ++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 8e9db8c..fede096 100644 --- a/README.rst +++ b/README.rst @@ -159,10 +159,12 @@ Parse apache log from remote server with `common` format Use the `email` for the porcessor-report the `5xx` error real-time(check every 10 second.): ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - $ ngxtop --filter '5 == status_type' \ - -e 'user_who_want_recevc_the_email@xxx.com;user2@163.com'\ - -F 'email_send_from' \ - -S 'smtp_server' \ - -u 'smtp_auth_user' \ - -P '****password***' \ - -t 10 \ No newline at end of file +:: + + $ ngxtop --filter '5 == status_type' \\ + -e 'user_who_want_recevc_the_email@xxx.com;user2@163.com'\\ + -F 'email_send_from' \\ + -S 'smtp_server' \\ + -u 'smtp_auth_user' \\ + -P '**password**' \\ + -t 10 \ No newline at end of file diff --git a/setup.py b/setup.py index 8854c22..476c8b0 100644 --- a/setup.py +++ b/setup.py @@ -2,14 +2,14 @@ setup( name='ngxtop', - version='0.0.2', + version='0.0.3', description='Real-time metrics for nginx server', long_description=open('README.rst').read(), license='MIT', - url='https://github.com/lebinh/ngxtop', + url='https://github.com/lebinh/ngxtop;https://github.com/toontong/ngxtop', author='Binh Le', - author_email='lebinh.it@gmail.com', + author_email='lebinh.it@gmail.com;chuantong.huang@gmail.com', classifiers=[ 'Development Status :: 4 - Beta', @@ -24,7 +24,7 @@ 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', ], - keywords='cli monitoring nginx system', + keywords='access-log cli monitoring nginx system', packages=['ngxtop'], install_requires=['docopt', 'tabulate', 'pyparsing'], From 5f0a209fb77eee7ae9b5d4821cc7c03486088a22 Mon Sep 17 00:00:00 2001 From: toontong Date: Fri, 16 May 2014 16:52:45 +0800 Subject: [PATCH 4/7] if curses.initscr() fail. if maybe no terminal, run as service. update version to 0.0.4 --- ngxtop/ngxtop.py | 11 ++++++++--- setup.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index b888d11..c4d7c1f 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -213,12 +213,17 @@ def build_source(access_log, arguments): def setup_reporter(processor, arguments): if arguments['--no-follow']: return - - scr = curses.initscr() - atexit.register(curses.endwin) + no_terminal = False + try: + scr = curses.initscr() + atexit.register(curses.endwin) + except curses.error: + no_terminal = True def print_report(sig, frame): output = processor.report() + if no_terminal: + return scr.erase() try: scr.addstr(output) diff --git a/setup.py b/setup.py index 476c8b0..89f3171 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='ngxtop', - version='0.0.3', + version='0.0.4', description='Real-time metrics for nginx server', long_description=open('README.rst').read(), license='MIT', From 16815c32c0bc3f045189005f43d14627c848e2e6 Mon Sep 17 00:00:00 2001 From: toontong Date: Mon, 19 May 2014 11:09:30 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=90=8D=E6=9C=89?= =?UTF-8?q?=E7=A9=BA=E6=A0=BC=E3=80=82=E6=B1=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ngxtop/{emailprocessor .py => emailprocessor.py} | 0 ngxtop/ngxtop.py | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) rename ngxtop/{emailprocessor .py => emailprocessor.py} (100%) diff --git a/ngxtop/emailprocessor .py b/ngxtop/emailprocessor.py similarity index 100% rename from ngxtop/emailprocessor .py rename to ngxtop/emailprocessor.py diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index c4d7c1f..df9e80d 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -64,7 +64,7 @@ Analyze apache access log from remote machine using 'common' log format $ ssh remote tail -f /var/log/apache2/access.log | ngxtop -f common """ -from __future__ import print_function +from __future__ import print_function, absolute_import import atexit from contextlib import closing import curses @@ -83,10 +83,10 @@ from docopt import docopt import tabulate -from config_parser import detect_log_config, detect_config_path, extract_variables, build_pattern -from utils import error_exit -from sqlprocessor import SQLProcessor -from emailprocessor import EMailProcessor +from .config_parser import detect_log_config, detect_config_path, extract_variables, build_pattern +from .utils import error_exit +from .sqlprocessor import SQLProcessor +from .emailprocessor import EMailProcessor # ====================== # generator utilities From 74d9c11f6b46d0c6d32b57403522c459b48bbfdf Mon Sep 17 00:00:00 2001 From: Huang ChuanTong Date: Tue, 10 Jun 2014 09:21:30 +0800 Subject: [PATCH 6/7] bug fix: send to mutil mails. bug fix: send to mutil mails. and add the day format email subject . --- ngxtop/emailprocessor.py | 23 +++++++++++++---------- ngxtop/ngxtop.py | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ngxtop/emailprocessor.py b/ngxtop/emailprocessor.py index 8597eac..2dd119e 100644 --- a/ngxtop/emailprocessor.py +++ b/ngxtop/emailprocessor.py @@ -11,16 +11,15 @@ class EMailProcessor(BaseProcessor): - #def __init__(self, report_queries, fields, index_fields=None, second=20): def __init__(self, arguments): self.emails_to = arguments['--email'] self.smtp = arguments['--smtp'] self.user = arguments['--user'] self.password = arguments['--password'] self.sender = arguments['--from'] - self.subject = '[%s]-%s' % (socket.gethostname(), arguments['--subject']) self.no_follow = arguments['--no-follow'] self.debug = arguments['--debug'] or arguments['--verbose'] + self.arguments = arguments fmt = arguments['--log-format'].replace('-', '') fmt = fmt.replace('[', '') @@ -57,7 +56,7 @@ def _make_report(self, summary, detail, access_log_buffer): lst.append(pprint.pformat(summary, indent=4)) lst.append(split % 'Detailed') lst.append(pprint.pformat(detail, indent=4)) - lst.append(split % 'Access Logs[ Limit 10]') + lst.append(split % 'Access Logs[Show in this mail, Max Limit 10]') lst.append(pprint.pformat(access_log_buffer[:10], indent=2)) return '\n'.join(lst) @@ -85,27 +84,31 @@ def report(self): return msg def _send_mail(self, content): - logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]', - self.subject, self.emails_to, self.smtp, self.user) + now = datetime.now() + subject = '[%s]-[%s]-%s' % (socket.gethostname(), now.strftime( '%Y-%m-%d'), self.arguments['--subject']) + + logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]', + subject, self.emails_to, self.smtp, self.user) + if not self.emails_to: + return False, 'emails_to is empty.' + msg = MIMEText(content, 'plain', _charset='utf-8') if self.debug: logging.debug('email content:\n%s', content) return True, 'just-test, email did not send.' - msg['Subject'] = self.subject + msg['Subject'] = subject msg['From'] = self.sender msg['To'] = self.emails_to try: s = smtplib.SMTP() s.connect(self.smtp) s.login(self.user, self.password) - s.sendmail(self.sender, self.emails_to, msg.as_string()) + s.sendmail(self.sender, self.emails_to.split(';'), msg.as_string()) s.close() logging.info('email was send to[%s]', self.emails_to) return True, 'success' except Exception, e: logging.error('send_mail to[%s] Exception[%s]', self.emails_to, e) - return False, 'fail' - - + return False, 'Exception raise.' diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index df9e80d..98d91e0 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -27,7 +27,7 @@ -u , --user smtp auth user. -P , --password smtp auth user's passwod. -F , --from who send the email. - -T , --subject email Subject-title [default: (hostname)ngxtop-access-log-email-notify]. + -T , --subject email Subject-title [default: Ngxtop-Access-Log-Email-Notify]. -v, --verbose more verbose output -d, --debug print every line and parsed record From eb43a30f17a5069ac50bbd7e05b249d40206107a Mon Sep 17 00:00:00 2001 From: Huang ChuanTong Date: Tue, 10 Jun 2014 09:36:08 +0800 Subject: [PATCH 7/7] bug fix: replace tab by space --- ngxtop/emailprocessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ngxtop/emailprocessor.py b/ngxtop/emailprocessor.py index 2dd119e..a022b4d 100644 --- a/ngxtop/emailprocessor.py +++ b/ngxtop/emailprocessor.py @@ -19,7 +19,7 @@ def __init__(self, arguments): self.sender = arguments['--from'] self.no_follow = arguments['--no-follow'] self.debug = arguments['--debug'] or arguments['--verbose'] - self.arguments = arguments + self.arguments = arguments fmt = arguments['--log-format'].replace('-', '') fmt = fmt.replace('[', '') @@ -87,10 +87,10 @@ def _send_mail(self, content): now = datetime.now() subject = '[%s]-[%s]-%s' % (socket.gethostname(), now.strftime( '%Y-%m-%d'), self.arguments['--subject']) - logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]', + logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]', subject, self.emails_to, self.smtp, self.user) - if not self.emails_to: - return False, 'emails_to is empty.' + if not self.emails_to: + return False, 'emails_to is empty.' msg = MIMEText(content, 'plain', _charset='utf-8') if self.debug: