diff --git a/lib/pavilion/parsers/expressions.py b/lib/pavilion/parsers/expressions.py index 37e69774a..12066e4bb 100644 --- a/lib/pavilion/parsers/expressions.py +++ b/lib/pavilion/parsers/expressions.py @@ -19,31 +19,31 @@ // All expressions will resolve to the start expression. start: expr _WS? - | // An empty string is valid - + | // An empty string is valid + // Trailing whitespace is ignored. Whitespace between tokens is // ignored below. _WS: /\s+/ expr: or_expr -// These set order of operations. +// These set order of operations. // See https://en.wikipedia.org/wiki/Operator-precedence_parser -or_expr: and_expr ( OR and_expr )* +or_expr: and_expr ( OR and_expr )* and_expr: not_expr ( AND not_expr )* -not_expr: NOT? compare_expr +not_expr: NOT? compare_expr compare_expr: add_expr ((EQ | NOT_EQ | LT | GT | LT_EQ | GT_EQ ) add_expr)* add_expr: mult_expr ((PLUS | MINUS) mult_expr)* mult_expr: pow_expr ((TIMES | DIVIDE | INT_DIV | MODULUS) pow_expr)* pow_expr: primary ("^" primary)? -primary: literal - | var_ref +primary: literal + | var_ref | negative | "(" expr ")" | function_call | list_ -// A function call can contain zero or more arguments. +// A function call can contain zero or more arguments. function_call: NAME "(" (expr ("," expr)*)? ")" negative: (MINUS|PLUS) primary @@ -53,7 +53,7 @@ | FLOAT | BOOL | ESCAPED_STRING - + // Allows for trailing commas list_: L_BRACKET (expr ("," expr)* ","?)? R_BRACKET @@ -92,11 +92,11 @@ // This will be prioritized over 'NAME' matches BOOL.2: "True" | "False" -// Names can be lower-case or capitalized, but must start with a letter or +// Names can be lower-case or capitalized, but must start with a letter or // underscore NAME.1: /[a-zA-Z_][a-zA-Z0-9_]*/ -// Ignore all whitespace between tokens. +// Ignore all whitespace between tokens. %ignore / +(?=[^.])/ ''' diff --git a/lib/pavilion/parsers/strings.py b/lib/pavilion/parsers/strings.py index 6325bda5f..3ba772857 100644 --- a/lib/pavilion/parsers/strings.py +++ b/lib/pavilion/parsers/strings.py @@ -15,13 +15,13 @@ from .expressions import get_expr_parser, ExprTransformer, VarRefVisitor STRING_GRAMMAR = r''' -// All strings resolve to this token. +// All strings resolve to this token. start: string TRAILING_NEWLINE? TRAILING_NEWLINE: /\n/ -// It's important that each of these start with a terminal, rather than -// a reference back to the 'string' rule. A 'STRING' terminal (or nothing) +// It's important that each of these start with a terminal, rather than +// a reference back to the 'string' rule. A 'STRING' terminal (or nothing) // is definite, but a 'string' would be non-deterministic. string: STRING? | STRING? iter string @@ -35,15 +35,15 @@ iter_inner: STRING? | STRING? expr iter_inner - + expr: _START_EXPR EXPR? (ESCAPED_STRING EXPR?)* FORMAT? _END_EXPR _START_EXPR: "{{" _END_EXPR: "}}" EXPR: /[^}~{":]+/ -// Match anything enclosed in quotes as long as the last +// Match anything enclosed in quotes as long as the last // escape doesn't escape the close quote. -// A minimal match, but the required close quote will force this to +// A minimal match, but the required close quote will force this to // consume most of the string. _STRING_ESC_INNER: /.*?/ // If the string ends in a backslash, it must end with an even number @@ -62,7 +62,7 @@ // we can't match the start of the string in the look-behind. // - Strings can contain anything, but they can't start with an open // expression '{{' or open iteration '[~'. -// - Strings cannot end in an odd number of backslashes (that would +// - Strings cannot end in an odd number of backslashes (that would // escape the closing characters). // - Strings must end with the end of string, an open expression '{{', // an open iteration '[~', or a tilde. diff --git a/lib/pavilion/resolver.py b/lib/pavilion/resolver.py index c82c2b951..b0670ad87 100644 --- a/lib/pavilion/resolver.py +++ b/lib/pavilion/resolver.py @@ -665,7 +665,7 @@ def _load_raw_config(self, name: str, config_type: str, optional=False) \ "Pavilion config directories.\n" "Run `pav show {2}` to get a list of available {0} files." .format(config_type, name, show_type)) - + try: with path.open() as cfg_file: # Load the host test config defaults. diff --git a/lib/pavilion/result/parse.py b/lib/pavilion/result/parse.py index ead75322b..684fd69fc 100644 --- a/lib/pavilion/result/parse.py +++ b/lib/pavilion/result/parse.py @@ -70,6 +70,31 @@ def __init__(self, parser_name: str, key: str, config: dict): ProcessFileArgs = NewType('ProcessFileArgs', Tuple[Path, List[KeySet]]) +def format_results(result_val, format_spec): + """Format the result value according to the format spec. + :param result_val: The value to format. + :param format_spec: The format spec. + :return: The formatted value. + """ + + #Check if result_val is a string or bool. + if isinstance(result_val, (str, bool)): + return result_val + + #Check if result_val is numeric. + if isinstance(result_val, (int, float)): + return format_spec.format(result_val) + + if isinstance(result_val, (list, set)): + formatted_result=[] + for res_v in result_val: + if isinstance(res_v, (int, float)): + formatted_result.append(format_spec.format(res_v)) + else: + formatted_result.append(res_v) + + return formatted_result + def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: """Parse the results of the given test using all the result parsers @@ -107,6 +132,8 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: per_file = {} # Action values by key actions = {} + # Format values by key + formats = {} # A list of encountered error messages. errors = [] @@ -120,6 +147,7 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: per_file[key] = rconf['per_file'] actions[key] = rconf['action'] + formats[key] = rconf['format'] for file_glob in rconf['files']: base_glob = file_glob @@ -190,7 +218,7 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: for key, per_file_name in per_file.items(): per_file_func = PER_FILES[per_file_name] # type: per_first action_name = actions[key] - presults = ordered_filed_results[key] + presults = format_results(ordered_filed_results[key], formats[key]) try: log("Applying per-file option '{}' and action '{}' to key '{}'." diff --git a/lib/pavilion/result_parsers/base_classes.py b/lib/pavilion/result_parsers/base_classes.py index c125fa6b6..1bbd61143 100644 --- a/lib/pavilion/result_parsers/base_classes.py +++ b/lib/pavilion/result_parsers/base_classes.py @@ -211,7 +211,7 @@ def check_args(self, **kwargs) -> dict: args[key] = kwargs[key] kwargs = args - base_keys = ('action', 'per_file', 'files', 'match_select', + base_keys = ('action', 'per_file', 'files', 'format', 'match_select', 'for_lines_matching', 'preceded_by') for key in base_keys: @@ -313,6 +313,10 @@ def check_args(self, **kwargs) -> dict: help_text="Path to the file/s that this result parser " "will examine. Each may be a file glob," "such as '*.log'"), + yc.StrElem( + "format", + help_text="Python string format 'eg. {:.3f}' to store the result." + ), yc.StrElem( "per_file", help_text=( @@ -470,6 +474,7 @@ def add_arg_doc(arg): _DEFAULTS = { 'per_file': PER_FIRST, 'action': ACTION_STORE, + 'format': '{:.3f}', 'files': ['../run.log'], 'match_select': MATCH_FIRST, 'for_lines_matching': '', diff --git a/test/utils/trimwstrail b/test/utils/trimwstrail new file mode 100755 index 000000000..e0272a5c1 --- /dev/null +++ b/test/utils/trimwstrail @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import os +import sys + +def trim_py_files(directories): + """Remove trailing whitespace on all .py files in the given directories. + """ + nchanged = 0 + print(directories) + for directory in directories: + for root, dirs, files in os.walk(directory): + if ".git" in root: + continue + for fname in files: + filename = os.path.join(root, fname) + if fname.endswith('.py'): + with open(filename, 'rb') as f: + code1 = f.read().decode() + lines = [line.rstrip() for line in code1.splitlines()] + while lines and not lines[-1]: + lines.pop(-1) + lines.append('') # always end with a newline + code2 = '\n'.join(lines) + if code1 != code2: + nchanged += 1 + print(' Removing trailing whitespace on', filename) + with open(filename, 'wb') as f: + f.write(code2.encode()) + print('Removed trailing whitespace on {} files.'.format(nchanged)) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + trim_py_files(list(os.path.dirname(__file__))) + else: + trim_py_files(sys.argv[1:]) +