diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 0d9f00a..17a4244 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -4,6 +4,8 @@ import os import re import subprocess +import glob + from pyparsing import Literal, Word, ZeroOrMore, OneOrMore, Group, \ printables, quotedString, pythonStyleComment, removeQuotes @@ -59,17 +61,21 @@ def get_access_logs(config): access_log = Literal("access_log") + ZeroOrMore(parameter) + semicolon access_log.ignore(pythonStyleComment) + access_logs_dict = {} for directive in access_log.searchString(config).asList(): path = directive[1] if path == 'off' or path.startswith('syslog:'): # nothing to process here continue - - format_name = 'combined' + access_logs_dict[path] = ['combined'] if len(directive) > 2 and '=' not in directive[2]: - format_name = directive[2] + if directive[2] not in access_logs_dict[path]: + if 'combined' in access_logs_dict[path]: + access_logs_dict[path] = [directive[2]] + else: + (access_logs_dict[path]).append(directive[2]) + return access_logs_dict - yield path, format_name def get_log_formats(config): @@ -86,6 +92,32 @@ def get_log_formats(config): format_string = ''.join(directive[2]) yield name, format_string +def get_config_str(config): + """ + Parse config for include_format directives + """ + # include path + include = Literal("include") + ZeroOrMore(parameter) + semicolon + include.ignore(pythonStyleComment) + + config_str = '' + config_str_list = [] + path_list = [config] + with open(config) as f1: + config_str_list.append(f1.read()) + for directive in include.searchString(config_str_list[0]).asList(): + path = directive[1] + path_list.append (path) + for path in path_list: + if path == config: + continue + file_list = glob.glob(path) + for file_path in file_list: + with open(file_path) as f2: + config_str_list.append(f2.read()) + config_str = '\n'.join(config_str_list) + return config_str + def detect_log_config(arguments): """ @@ -98,53 +130,65 @@ def detect_log_config(arguments): if not os.path.exists(config): error_exit('Nginx config file not found: %s' % config) - with open(config) as f: - config_str = f.read() - access_logs = dict(get_access_logs(config_str)) - if not access_logs: + config_str = get_config_str(config) + access_logs_dict = get_access_logs(config_str) + if len(access_logs_dict) == 0: error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config) - log_formats = dict(get_log_formats(config_str)) - if len(access_logs) == 1: - log_path, format_name = list(access_logs.items())[0] - if format_name == 'combined': - return log_path, LOG_FORMAT_COMBINED - if format_name not in log_formats: - error_exit('Incorrect format name set in config for access log file "%s"' % log_path) - return log_path, log_formats[format_name] + log_formats_dict = dict(get_log_formats(config_str)) + if len(access_logs_dict) == 1: + for log_path in access_logs_dict: + if access_logs_dict[log_path] == ['combined']: + log_formats_dict.clear() + log_formats_dict['combined'] = LOG_FORMAT_COMBINED + return log_path, log_formats_dict + for format_name in access_logs_dict[log_path]: + if format_name not in log_formats_dict: + error_exit('Incorrect format name set in config for access log file "%s"' % log_path) + return log_path, log_formats_dict # multiple access logs configured, offer to select one print('Multiple access logs detected in configuration:') - log_path = choose_one(list(access_logs.keys()), 'Select access log file to process: ') - format_name = access_logs[log_path] - if format_name not in log_formats: - error_exit('Incorrect format name set in config for access log file "%s"' % log_path) - return log_path, log_formats[format_name] + log_path = choose_one(list(access_logs_dict.keys()), 'Select access log file to process: ') + format_name_list = access_logs_dict[log_path] + for format_name in format_name_list: + if format_name not in log_formats_dict: + error_exit('Incorrect format name set in config for access log file "%s"' % log_path) + return log_path, dict(log_formats_dict[log_path]) -def build_pattern(log_format): +def build_pattern(log_formats_dict, arguments): """ Build regular expression to parse given format. :param log_format: format string to parse :return: regular expression to parse given format """ - if log_format == 'combined': - log_format = LOG_FORMAT_COMBINED - elif log_format == 'common': - log_format = LOG_FORMAT_COMMON - pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) - pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) - return re.compile(pattern) - - -def extract_variables(log_format): + log_format = arguments['--log-format'] + if len(log_formats_dict) == 0: + if log_format == 'combined': + log_format = LOG_FORMAT_COMBINED + elif log_format == 'common': + log_format = LOG_FORMAT_COMMON + pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) + pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) + return re.compile(pattern) + else: + pattern_list = [] + for key in log_formats_dict: + pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_formats_dict[key]) + pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) + pattern_list.append(re.compile(pattern)) + return pattern_list + +def extract_variables(log_formats_dict): """ Extract all variables from a log format string. :param log_format: format string to extract :return: iterator over all variables in given format string """ - if log_format == 'combined': - log_format = LOG_FORMAT_COMBINED - for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format): - yield match + for log_format in log_formats_dict: + if log_format == 'combined': + log_format = LOG_FORMAT_COMBINED + for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format): + yield match diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 8667b8b..e356372 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -183,7 +183,10 @@ def to_float(value): def parse_log(lines, pattern): - matches = (pattern.match(l) for l in lines) + if isinstance(pattern, list): + matches = (pattern_value.match(l) for l in lines for pattern_value in pattern) + else: + matches = (pattern.match(l) for l in lines ) records = (m.groupdict() for m in matches if m is not None) records = map_field('status', to_int, records) records = add_field('status_type', parse_status_type, records) @@ -257,7 +260,6 @@ def process_log(lines, pattern, processor, arguments): pre_filer_exp = arguments['--pre-filter'] if pre_filer_exp: lines = (line for line in lines if eval(pre_filer_exp, {}, dict(line=line))) - records = parse_log(lines, pattern) filter_exp = arguments['--filter'] @@ -344,27 +346,27 @@ def print_report(sig, frame): def process(arguments): access_log = arguments['--access-log'] - log_format = arguments['--log-format'] + log_formats_dict = {} if access_log is None and not sys.stdin.isatty(): # assume logs can be fetched directly from stdin when piped access_log = 'stdin' if access_log is None: - access_log, log_format = detect_log_config(arguments) + access_log, log_formats_dict = detect_log_config(arguments) + logging.info('log_format: %s', log_formats_dict) logging.info('access_log: %s', access_log) - logging.info('log_format: %s', log_format) if access_log != 'stdin' and not os.path.exists(access_log): error_exit('access log file "%s" does not exist' % access_log) if arguments['info']: print('nginx configuration file:\n ', detect_config_path()) print('access log file:\n ', access_log) - print('access log format:\n ', log_format) - print('available variables:\n ', ', '.join(sorted(extract_variables(log_format)))) + print('access log format:\n ', log_formats_dict) + print('available variables:\n ', ', '.join(sorted(extract_variables(log_formats_dict)))) return source = build_source(access_log, arguments) - pattern = build_pattern(log_format) + pattern = build_pattern(log_formats_dict, arguments) processor = build_processor(arguments) setup_reporter(processor, arguments) process_log(source, pattern, processor, arguments)