add argument support: -s <number>, --second <number> seconds of the reco...

README.rst

@@ -32,19 +32,28 @@ Usage
         ngxtop info
 
     Options:
+        -r <reportor>, --reportor <reportor>  choice in[sql, email] or other Implemented by your self(see the source code sqlprocessor.py). [default: sql]
         -l <file>, --access-log <file>  access log file to parse.
-        -f <format>, --log-format <format>  log format as specify in log_format directive.
+        -f <format>, --log-format <format>  log format as specify in log_format directive. [default: combined]
         --no-follow  ngxtop default behavior is to ignore current lines in log
                          and only watch for new lines as they are written to the access log.
                          Use this flag to tell ngxtop to process the current content of the access log instead.
         -t <seconds>, --interval <seconds>  report interval when running in follow mode [default: 2.0]
 
-        -g <var>, --group-by <var>  group by variable [default: request_path]
+        -g <var>, --group-by <var>  group by variable [default: request_path].
         -w <var>, --having <expr>  having clause [default: 1]
         -o <var>, --order-by <var>  order of output for default query [default: count]
         -n <number>, --limit <number>  limit the number of records included in report for top command [default: 10]
+        -s <number>, --second <number> seconds of the records save in the memory [default: 20]
         -a <exp> ..., --a <exp> ...  add exp (must be aggregation exp: sum, avg, min, max, etc.) into output
-
+
+        -e <email>, --email <email> ...  email to who.
+        -S <smtp>, --smtp <smtp>  smtp service.
+        -u <user>, --user <user>  smtp auth user.
+        -P <password>, --password <password>  smtp auth user's passwod.
+        -F <from-mail>, --from <from-email>  who send the email.
+        -T <subject>, --subject <subject>  email Subject-title [default: (hostname)ngxtop-access-log-email-notify].
+
         -v, --verbose  more verbose output
         -d, --debug  print every line and parsed record
         -h, --help  print this help message.
@@ -147,3 +156,15 @@ Parse apache log from remote server with `common` format
     | /xxxxx/                                  |      14 |            0.000 |     0 |    14 |     0 |     0 |
     | /xxxxxxxxxx/xxxxxxxx/xxxxx               |      13 |        20530.154 |    13 |     0 |     0 |     0 |
 
+Use the `email` for the porcessor-report the `5xx` error real-time(check every 10 second.):
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    $ ngxtop --filter '5 == status_type' \\
+    -e '[email protected];[email protected]'\\
+    -F 'email_send_from' \\
+    -S 'smtp_server' \\
+    -u 'smtp_auth_user' \\
+    -P '**password**' \\
+    -t 10 

ngxtop/emailprocessor.py

@@ -0,0 +1,114 @@
+import time
+import socket
+import pprint
+import logging
+import smtplib
+from datetime import datetime
+from email.mime.text import MIMEText
+
+import tabulate
+from processor import BaseProcessor
+
+
+class EMailProcessor(BaseProcessor):
+    def __init__(self, arguments):
+        self.emails_to = arguments['--email']
+        self.smtp = arguments['--smtp']
+        self.user = arguments['--user']
+        self.password = arguments['--password']
+        self.sender = arguments['--from']
+        self.no_follow = arguments['--no-follow']
+        self.debug = arguments['--debug'] or arguments['--verbose']
+        self.arguments = arguments
+
+        fmt = arguments['--log-format'].replace('-', '')
+        fmt = fmt.replace('[', '')
+        fmt = fmt.replace(']', '')
+        fmt = fmt.replace('$', '')
+
+        self.logfmtkeys = [k for k in fmt.split(' ') if k]
+        self.begin = 0
+        self.access_log_buffer = []
+        self.summary = {}
+        self.detail = {}
+
+    def process(self, records):
+        self.begin = time.time()
+        for r in records:
+           self.access_log_buffer.append(r)
+           try:
+               status_code_key = '%sxx' % r['status_type']
+               self.summary[status_code_key] = 1 + self.summary.setdefault(status_code_key, 0)
+               self.summary['count'] = 1 + self.summary.setdefault('count', 0)
+
+               path = r['request_path']
+               path_info = self.detail.setdefault(path, {})
+               path_info['count'] = 1 + path_info.setdefault('count', 0)
+               path_info[r['status'] ] = 1 + path_info.setdefault(r['status'] , 0)
+
+           except Exception as e:
+               logging.warning('log-record can not parse.[%s]. Exception[%s]', r, e)
+
+    def _make_report(self, summary, detail, access_log_buffer):
+        title = '************ host[%s] date[%s] **************' % (socket.gethostname(), datetime.now())
+        split = '\n------------------- %s --------------------\n'
+        lst = [title, split % 'Summary']
+        lst.append(pprint.pformat(summary, indent=4))
+        lst.append(split % 'Detailed')
+        lst.append(pprint.pformat(detail, indent=4))
+        lst.append(split % 'Access Logs[Show in this mail, Max Limit 10]')
+        lst.append(pprint.pformat(access_log_buffer[:10], indent=2)) 
+        return '\n'.join(lst)
+
+    def report(self):
+        if not self.begin:
+           logging.warning('process did not begin.')
+           return ''
+
+        if not self.access_log_buffer:
+           logging.debug('access-log buffer is empty.')
+           return 'access-log buffer is empty.'
+
+        summary, detail, access_log_buffer = self.summary, self.detail, self.access_log_buffer
+
+        hr, res = self._send_mail(self._make_report(summary, detail, access_log_buffer))
+
+        if hr:
+            self.summary, self.detail, self.access_log_buffer = {}, {}, []
+
+        msg = '[%s] send report to then email[%s] --> [%s].' % (datetime.now(), self.emails_to, res)
+
+        if self.no_follow:
+            print(msg)
+        else:
+            return msg
+
+    def _send_mail(self, content):
+        now = datetime.now() 
+        subject = '[%s]-[%s]-%s' % (socket.gethostname(), now.strftime( '%Y-%m-%d'), self.arguments['--subject'])
+
+        logging.info('will send email[%s] to[%s],smtp[%s]-user[%s]',
+                     subject, self.emails_to, self.smtp, self.user)
+        if not self.emails_to:
+            return False, 'emails_to is empty.'
+
+        msg = MIMEText(content, 'plain', _charset='utf-8')
+        if self.debug:
+            logging.debug('email content:\n%s', content)
+            return True, 'just-test, email did not send.'
+
+        msg['Subject'] = subject
+        msg['From'] = self.sender
+        msg['To'] = self.emails_to
+        try:
+            s = smtplib.SMTP()
+            s.connect(self.smtp)
+            s.login(self.user, self.password)
+            s.sendmail(self.sender, self.emails_to.split(';'), msg.as_string())
+            s.close()
+            logging.info('email was send to[%s]', self.emails_to)
+            return True, 'success'
+        except Exception, e:
+            logging.error('send_mail to[%s] Exception[%s]',
+                          self.emails_to, e)
+            return False, 'Exception raise.'