[v0.9.0] Added file filter feature

exetest/env_vars.py

@@ -12,3 +12,4 @@ class ExeTestEnvVars:
     EXETEST_DIFF_EXE = 'EXETEST_DIFF_EXE'
     NO_RUN = 'EXETEST_NO_RUN'
     DISABLE_SKIP = 'EXETEST_DISABLE_SKIP'
+    FILE_FILTER = 'EXETEST_FILE_FILTER'

exetest/exetest_decorator.py

@@ -1,6 +1,7 @@
 import os
 import os.path
 import pathlib
+import traceback
 from . import misc_utils
 from .misc_utils import working_dir, rmdir
 from .diff_utils import FileComparator, diff_dirs
@@ -107,7 +108,7 @@ def get_test_exe(val):
         self.ref_diff_only = ref_diff_only
 
         self.test_name = ''
-        self.comparators = comparators if comparators is not None else {}
+        self.comparators = comparators or {}
         self.exception_handler = exception_handler
         self.common_env_vars = env_vars or dict()
         self.pre_cmd = pre_cmd if pre_cmd else []
@@ -127,6 +128,7 @@ def get_test_exe(val):
         self._nested_out_dir = nested_out_dir
         self._keep_output_on_success = True if ExeTestEnvVars.KEEP_OUTPUT_ON_SUCCESS in os.environ else keep_output_on_success
         self._num_lines_diff = int(os.environ.get(ExeTestEnvVars.NUM_DIFFS, 20))
+        self._file_filter = os.environ.get(ExeTestEnvVars.FILE_FILTER, '').split('+')
 
     @staticmethod
     def get_test_subdir(test_name):
@@ -269,33 +271,16 @@ def run_test(self, exe_args, compare_spec,
             if compare_spec and not files_to_compare.items():
                 self.raise_exception(f"No reference output files for {compare_spec}")
 
-            created_dirs = []
-            for ref_file, new_file in files_to_compare.items():
-
-                if os.path.isdir(ref_file):
-                    file_dir = new_file
-                else:
-                    file_dir = os.path.dirname(new_file)
-
-                if file_dir and not os.path.exists(file_dir):
-                    topmost_created_dir = file_dir
-                    while True:
-                        parent_dir = os.path.split(topmost_created_dir)[0]
-                        if not parent_dir or parent_dir == self.TMP_OUTPUT_DIR or os.path.exists(parent_dir):
-                            break
-                        else:
-                            topmost_created_dir = parent_dir
-
-                    os.makedirs(file_dir, exist_ok=True)
-                    created_dirs.append(topmost_created_dir)
-
             tmp_output_dir = self.get_out_dir(test_name)
             run_from_dir = os.path.join(self.test_root, tmp_output_dir) \
                 if self.run_from_out_dir else self.test_root
 
         if not self._compare_only:
             self.clear_dir(tmp_output_dir, recreate=True)
 
+        created_dirs = []
+        os.makedirs(tmp_output_dir, exist_ok=True)
+
         with working_dir(run_from_dir):
             try:
                 misc_utils.exec_cmdline(self.exe_path,
@@ -354,6 +339,8 @@ def run_compare(self, files_to_compare):
                     self.raise_exception(f"Missing reference file: {ref_file} - "
                                          f"you can rebase with --rebase option")
                 if not has_new:
+                    while not os.path.exists(os.path.dirname(new_file)):
+                        new_file = os.path.dirname(new_file)
                     self.raise_exception(f"Missing output file: {new_file}")
 
         num_diffs = 0
@@ -473,7 +460,7 @@ def compare_equal(self, ref_file, new_file, throw=True):
 
             if compare_equal:
                 if self.verbose:
-                    print(f'files match{comparison_description}: {files_info}')
+                    print(f'files match{comparison_description}: {ref_file}')
             else:
                 all_equal = False
                 error_msg = f'files differ{comparison_description}: {files_info}'
@@ -513,7 +500,7 @@ def get_files_to_compare(self, test_name, compare_spec=None) -> dict:
                 return {ref_dir: out_dir}
 
         elif not compare_spec:
-            return []
+            return {}
 
         if isinstance(compare_spec, str):
             # single reference file
@@ -537,10 +524,12 @@ def get_files_to_compare(self, test_name, compare_spec=None) -> dict:
                 ref_path = os.path.normpath(ref_path)
                 new_path = os.path.normpath(new_path)
 
-                for dirpath, dirnames, filenames in os.walk(ref_path):
+                for dirpath, dirnames, filenames in os.walk(ref_path, followlinks=True):
                     tmp_path = dirpath.replace(ref_path, new_path, 1)
                     for filename in filenames:
-                        files_to_compare[os.path.join(dirpath, filename)] = os.path.join(tmp_path, filename)
+                        ref_filepath = os.path.join(dirpath, filename)
+                        new_filepath = os.path.join(tmp_path, filename)
+                        files_to_compare[ref_filepath] = new_filepath
 
                 if not self.ref_diff_only:
                     for dirpath, dirnames, filenames in os.walk(new_path):
@@ -555,12 +544,17 @@ def get_files_to_compare(self, test_name, compare_spec=None) -> dict:
                 # self.raise_exception(ref_path.replace(ref_dir, new_path))
                 files_to_compare[ref_path] = ref_path.replace(ref_dir, new_path)
 
+        excluded_files = []
+        if self._file_filter:
+            for file in files_to_compare:
+                if not misc_utils.pattern_matches(pattern=self._file_filter, path_string=file):
+                    excluded_files.append(file)
+
         patterns_to_ignore = []
         for key, value in self.comparators.items():
             if value is None:
                 patterns_to_ignore.append(key)
 
-        excluded_files = []
         for file in files_to_compare:
             file_path = pathlib.PurePath(file)
             for pattern in patterns_to_ignore:

exetest/misc_utils.py

@@ -4,6 +4,8 @@
 import time
 from contextlib import contextmanager
 import subprocess
+import typing
+import pathlib
 
 
 def print_test_meta(*message):
@@ -157,6 +159,7 @@ def exec_cmdline(command, args_list, check_ret_code=True,
     :param pre_cmd:
     :param post_cmd:
     :param verbose:
+    :param norun:
     :return:
     """
 
@@ -221,3 +224,39 @@ def exec_cmdline(command, args_list, check_ret_code=True,
             raise
 
     return cmd_line
+
+
+def pattern_matches(pattern: typing.Union[str, typing.List[str]],
+                    path_string: str) -> bool:
+
+    if isinstance(pattern, str):
+        patterns = [pattern]
+    else:
+        patterns = pattern
+
+    has_suppress = False
+    has_keep = False
+    keep = None
+
+    def match(pattern, string):
+        if '*' in pattern:
+            return pathlib.PurePath(string).match(pattern)
+        else:
+            return pattern in string
+
+    for pattern in patterns:
+        if pattern.startswith('~'):
+            has_suppress = True
+            if match(pattern=pattern[1:], string=path_string):
+                keep = False
+
+        else:
+            has_keep = True
+            if match(pattern=pattern, string=path_string):
+                keep = True
+
+    if keep is None:
+        # default
+        keep = has_suppress and not has_keep
+
+    return keep

exetest/runmain.py

@@ -26,6 +26,10 @@ def main(prog, description=''):
                         const='',
                         type=str)
 
+    parser.add_argument("-f", "--file-filter",
+                        help="filter files to rebase / compare",
+                        nargs="+")
+
     parser.add_argument("--verbose",
                         help="verbose flag",
                         action='store_true')
@@ -118,6 +122,8 @@ def process_args(args, other_pytest_args):
             env_vars[ExeTestEnvVars.COMPARE_ONLY] = ''
         if args.keep_output:
             env_vars[ExeTestEnvVars.KEEP_OUTPUT_ON_SUCCESS] = ''
+        if args.file_filter:
+            env_vars[ExeTestEnvVars.FILE_FILTER] = '+'.join(args.file_filter)
 
         command += ['-v']
         verbose = len(args.test_cases) > 0 or args.verbose

setup.py

@@ -2,7 +2,7 @@
 setup(
   name='exetest',         # How you named your package folder (MyLib)
   packages=['exetest'],   # Chose the same as "name"
-  version='0.8.7',      # Start with a small number and increase it with every change you make
+  version='0.9.0',      # Start with a small number and increase it with every change you make
   license='MIT',        # Chose a license from here: https://help.github.com/articles/licensing-a-repository
   description='A pytest-based test framework for black-box approach to testing executables',   # Give a short description about your library
   author='Guillaume227',                   # Type in your name
@@ -18,7 +18,7 @@
     'Topic :: Software Development :: Build Tools',
     'Operating System :: OS Independent',
     'License :: OSI Approved :: MIT License',   # Again, pick a license
-    'Programming Language :: Python :: 3',      #Specify which pyhton versions that you want to support
+    'Programming Language :: Python :: 3',      #Specify the python versions that you want to support
   ],
   python_requires='>=3.6'
 )