From 7213bfd25c6004fd6005e2c28966bc898b9142cf Mon Sep 17 00:00:00 2001 From: stdweird Date: Thu, 8 Oct 2015 11:04:21 +0200 Subject: [PATCH 1/8] Support unittesting via 'python setup.py test' --- lib/vsc/install/shared_setup.py | 91 +++++++++++++++++++++++++++------ setup.py | 17 ++---- test/runner.py | 18 ++++--- 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index cdd494f3..06a26285 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -20,23 +20,46 @@ @author: Andy Georges (Ghent University) """ import glob +import inspect import os import shutil import sys import re + from distutils import log # also for setuptools from distutils.dir_util import remove_tree +from setuptools.command.test import test as TestCommand + + # 0 : WARN (default), 1 : INFO, 2 : DEBUG log.set_verbosity(2) has_setuptools = None +# available authors +ag = ('Andy Georges', 'andy.georges@ugent.be') +jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') +kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') +lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') +sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') +wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') +kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') +eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') +wp = ('Ward Poelmans', 'Ward.Poelmans@UGent.be') + +# FIXME: do we need this here? it won;t hurt, but still ... +REGEXP_REMOVE_SUFFIX = re.compile(r'(\.(?:py|sh|pl))$') # We do need all setup files to be included in the source dir if we ever want to install # the package elsewhere. EXTRA_SDIST_FILES = ['setup.py'] +# Put unittests under this directory +DEFAULT_TEST_SUITE = 'test' + +URL_GH_HPCUGENT = 'https://github.com/hpcugent/%(name)s' +URL_GHUGENT_HPCUGENT = 'https://github.ugent.be/hpcugent/%(name)s' def find_extra_sdist_files(): """Looks for files to append to the FileList that is used by the egg_info.""" @@ -122,20 +145,6 @@ class vsc_bdist_rpm_egg_info(vsc_egg_info): has_setuptools = False -# available authors -ag = ('Andy Georges', 'andy.georges@ugent.be') -jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') -kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') -lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') -sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') -wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') -kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') -eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') - - -# FIXME: do we need this here? it won;t hurt, but still ... -REGEXP_REMOVE_SUFFIX = re.compile(r'(\.(?:py|sh|pl))$') - class vsc_install_scripts(install_scripts): """Create the (fake) links for mympirun also remove .sh and .py extensions from the scripts.""" @@ -166,13 +175,56 @@ def find_package_modules (self, package, package_dir): class vsc_bdist_rpm(bdist_rpm): - """ Custom class to build the RPM, since the __inti__.py cannot be included for the packages that have namespace spread across all of the machine.""" + """Custom class to build the RPM, since the __init__.py cannot be included for the packages that have namespace spread across all of the machine.""" def run(self): log.error("vsc_bdist_rpm = %s" % (self.__dict__)) SHARED_TARGET['cmdclass']['egg_info'] = vsc_bdist_rpm_egg_info # changed to allow removal of files self.run_command('egg_info') # ensure distro name is up-to-date orig_bdist_rpm.run(self) +class VscTestCommand(TestCommand): + """ + The cmdclass for testing + """ + def run_tests(self): + # should be setup.py + setup_py = inspect.stack()[-1][1] + log.info('run_tests from %s' % setup_py) + base_dir = os.path.dirname(setup_py) + + # make a lib dir to trick setup.py to package this properly + # and git ignore empty dirs, so recreate it if necessary + lib_dir = os.path.abspath(os.path.join(base_dir, 'lib')) + if not os.path.exists(lib_dir): + os.mkdir(lib_dir) + + test_dir = os.path.abspath(os.path.join(base_dir, DEFAULT_TEST_SUITE)) + if os.path.isdir(test_dir): + sys.path.insert(0, test_dir) + else: + raise Exception("Can't find location of testsuite directory %s in %s" % (DEFAULT_TEST_SUITE, base_dir)) + + # make sure we can import the script as a module + scripts_dir = os.path.abspath(os.path.join(base_dir, 'bin')) + if os.path.isdir(scripts_dir): + sys.path.insert(0, scripts_dir) + + # and now the ugly bits: cleanup and restore vsc namespace + # we use vsc namespace tools very early + # need to make sure they are picked up from the paths as specified above + # not to mix with installed and already loaded modules + loaded_vsc_modules = [mod for mod in sys.modules.keys() if mod == 'vsc' or mod.startswith('vsc.')] + reload_vsc_modules = [] + for mod in loaded_vsc_modules: + if hasattr(sys.modules[mod], '__file__'): + # only actual modules + reload_vsc_modules.append(mod) + del(sys.modules[mod]) + # reimport + for mod in reload_vsc_modules: + __import__(mod) + + TestCommand.run_tests(self) # shared target config SHARED_TARGET = { @@ -184,6 +236,8 @@ def run(self): "egg_info": vsc_egg_info, "bdist_rpm": vsc_bdist_rpm, }, + 'cmdclass': {'test': VscTestCommand}, + 'test_suite': DEFAULT_TEST_SUITE, } @@ -265,7 +319,7 @@ def build_setup_cfg_for_bdist_rpm(target): setup_cfg.close() -def action_target(target, setupfn=setup, extra_sdist=[]): +def action_target(target, setupfn=setup, extra_sdist=[], urltemplate=None): # EXTRA_SDIST_FILES.extend(extra_sdist) do_cleanup = True @@ -283,6 +337,11 @@ def action_target(target, setupfn=setup, extra_sdist=[]): if do_cleanup: cleanup() + if urltemplate: + target['url'] = urltemplate % target + if 'github' in urltemplate: + target['download_url'] = "%s/tarball/master" % target['url'] + build_setup_cfg_for_bdist_rpm(target) x = parse_target(target) diff --git a/setup.py b/setup.py index 0f02ad79..227d88ce 100755 --- a/setup.py +++ b/setup.py @@ -32,35 +32,26 @@ @author: Andy Georges (Ghent University) @author: Kenneth Hoste (Ghent University) """ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) - import vsc.install.shared_setup as shared_setup -from vsc.install.shared_setup import ag, kh, jt, sdw +from vsc.install.shared_setup import ag, kh, jt, sdw, URL_GH_HPCUGENT def remove_bdist_rpm_source_file(): """List of files to remove from the (source) RPM.""" return [] shared_setup.remove_extra_bdist_rpm_files = remove_bdist_rpm_source_file -shared_setup.SHARED_TARGET.update({ - 'url': 'https://github.com/hpcugent/vsc-base', - 'download_url': 'https://github.com/hpcugent/vsc-base', - 'zip_safe': True, -}) PACKAGE = { 'name': 'vsc-base', - 'version': '2.3.0', + 'version': '2.3.1', 'author': [sdw, jt, ag, kh], 'maintainer': [sdw, jt, ag, kh], 'packages': ['vsc', 'vsc.install', 'vsc.utils'], 'scripts': ['bin/logdaemon.py', 'bin/startlogdaemon.sh', 'bin/bdist_rpm.sh', 'bin/optcomplete.bash'], 'install_requires' : ['setuptools'], + 'zip_safe': True, } if __name__ == '__main__': - shared_setup.action_target(PACKAGE) + shared_setup.action_target(PACKAGE, urltemplate=URL_GH_HPCUGENT) diff --git a/test/runner.py b/test/runner.py index 2e087257..1d8bd763 100644 --- a/test/runner.py +++ b/test/runner.py @@ -2,6 +2,9 @@ import os import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + import test.asyncprocess as a import test.dateandtime as td import test.docs as tdo @@ -22,11 +25,12 @@ suite = unittest.TestSuite([x.suite() for x in (a, td, tg, tf, te, tm, trest, trun, tt, topt, wrapt, tdo)]) -try: - import xmlrunner - rs = xmlrunner.XMLTestRunner(output="test-reports").run(suite) -except ImportError, err: - rs = unittest.TextTestRunner().run(suite) +if __name__ == '__main__': + try: + import xmlrunner + rs = xmlrunner.XMLTestRunner(output="test-reports").run(suite) + except ImportError, err: + rs = unittest.TextTestRunner().run(suite) -if not rs.wasSuccessful(): - sys.exit(1) + if not rs.wasSuccessful(): + sys.exit(1) From 480b2d0d146f7f67e35c655fe8ae9ea25c048c41 Mon Sep 17 00:00:00 2001 From: stdweird Date: Thu, 8 Oct 2015 11:23:14 +0200 Subject: [PATCH 2/8] restore sys.path hack to be able to use vsc.install --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index 227d88ce..c2a3072c 100755 --- a/setup.py +++ b/setup.py @@ -32,6 +32,16 @@ @author: Andy Georges (Ghent University) @author: Kenneth Hoste (Ghent University) """ + +# vsc-base setup.py needs vsc.install, which is currently shipped as part of vsc-base +# vsc.install doesn't require vsc-base, so we could move it to it's own repo and only +# have this hack in the setup.py of vsc.install (and set it as build_requires) +# until then... +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) + + import vsc.install.shared_setup as shared_setup from vsc.install.shared_setup import ag, kh, jt, sdw, URL_GH_HPCUGENT From 96bac0eb395fb8dce18581fb3486a8de8aadf18c Mon Sep 17 00:00:00 2001 From: stdweird Date: Thu, 8 Oct 2015 12:58:54 +0200 Subject: [PATCH 3/8] Address remarks --- lib/vsc/install/shared_setup.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index 06a26285..bd37bebf 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -20,7 +20,6 @@ @author: Andy Georges (Ghent University) """ import glob -import inspect import os import shutil import sys @@ -39,13 +38,13 @@ # available authors ag = ('Andy Georges', 'andy.georges@ugent.be') +eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') +kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') -kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') -eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') wp = ('Ward Poelmans', 'Ward.Poelmans@UGent.be') # FIXME: do we need this here? it won;t hurt, but still ... @@ -188,24 +187,24 @@ class VscTestCommand(TestCommand): """ def run_tests(self): # should be setup.py - setup_py = inspect.stack()[-1][1] + setup_py = os.path.abspath(sys.argv[0]) log.info('run_tests from %s' % setup_py) base_dir = os.path.dirname(setup_py) # make a lib dir to trick setup.py to package this properly # and git ignore empty dirs, so recreate it if necessary - lib_dir = os.path.abspath(os.path.join(base_dir, 'lib')) + lib_dir = os.path.join(base_dir, 'lib') if not os.path.exists(lib_dir): os.mkdir(lib_dir) - test_dir = os.path.abspath(os.path.join(base_dir, DEFAULT_TEST_SUITE)) + test_dir = os.path.join(base_dir, DEFAULT_TEST_SUITE) if os.path.isdir(test_dir): sys.path.insert(0, test_dir) else: raise Exception("Can't find location of testsuite directory %s in %s" % (DEFAULT_TEST_SUITE, base_dir)) # make sure we can import the script as a module - scripts_dir = os.path.abspath(os.path.join(base_dir, 'bin')) + scripts_dir = os.path.join(base_dir, 'bin') if os.path.isdir(scripts_dir): sys.path.insert(0, scripts_dir) From 64cdfee840bf79ed8cb2b59bc0c0ebb0dd8d930e Mon Sep 17 00:00:00 2001 From: stdweird Date: Thu, 8 Oct 2015 17:29:33 +0200 Subject: [PATCH 4/8] Add test filters: -F filters testmodules, -f filters test functions --- lib/vsc/install/shared_setup.py | 81 ++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index bd37bebf..af6ba61e 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -25,11 +25,16 @@ import sys import re +try: + from unittest.suite import TestSuite +except ImportError: + from unittest import TestSuite + from distutils import log # also for setuptools from distutils.dir_util import remove_tree from setuptools.command.test import test as TestCommand - +from setuptools.command.test import ScanningLoader # 0 : WARN (default), 1 : INFO, 2 : DEBUG log.set_verbosity(2) @@ -181,11 +186,85 @@ def run(self): self.run_command('egg_info') # ensure distro name is up-to-date orig_bdist_rpm.run(self) + +# private class variables to communicate +# between VscScanningLoader and VscTestCommand +# stored in __builtin__ because the .run_tests +# reload and cleanup in the modules +import __builtin__ +if not hasattr(__builtin__,'__test_filter'): + setattr(__builtin__, '__test_filter', { + 'module': None, + 'function': None, + 'allowmods': [], + }) + + +def filter_testsuites(testsuites): + """(Recursive) filtering of (suites of) tests""" + test_filter = getattr(__builtin__,'__test_filter')['function'] + + res = type(testsuites)() + + for t in testsuites: + # t is either a test or testsuite of more tests + if isinstance(t, TestSuite): + res.addTest(filter_testsuites(t)) + else: + if re.search(test_filter, t._testMethodName): + res.addTest(t) + return res + + +class VscScanningLoader(ScanningLoader): + def loadTestsFromModule(self, module): + """Allow for filter""" + testsuites = ScanningLoader.loadTestsFromModule(self, module) + test_filter = getattr(__builtin__,'__test_filter') + + res = testsuites + + if test_filter['module'] is not None: + mod = module.__name__ + if mod in test_filter['allowmods']: + # a parent name space + pass + elif re.search(test_filter['module'], mod): + if test_filter['function'] is not None: + res = filter_testsuites(testsuites) + # add parents (and module itself) + pms = mod.split('.') + for pm_idx in range(len(pms)): + pm = '.'.join(pms[:pm_idx]) + if not pm in test_filter['allowmods']: + test_filter['allowmods'].append(pm) + else: + res = type(testsuites)() + + return res + + class VscTestCommand(TestCommand): """ The cmdclass for testing """ + user_options = TestCommand.user_options + [ + ('test-filterf=', 'f', "Regex filter on test function names"), + ('test-filterm=', 'F', "Regex filter on test (sub)modules"), + ] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.test_filterm = None + self.test_filterf = None + self.test_loader = "vsc.install.shared_setup:VscScanningLoader" + def run_tests(self): + getattr(__builtin__,'__test_filter').update({ + 'function': self.test_filterf, + 'module': self.test_filterm, + }) + # should be setup.py setup_py = os.path.abspath(sys.argv[0]) log.info('run_tests from %s' % setup_py) From dc2f184ea32b852c6c4037f5f6156db2a3ed2138 Mon Sep 17 00:00:00 2001 From: stdweird Date: Fri, 9 Oct 2015 09:55:53 +0200 Subject: [PATCH 5/8] bye bye runner, you won't be missed --- test/runner.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 test/runner.py diff --git a/test/runner.py b/test/runner.py deleted file mode 100644 index 1d8bd763..00000000 --- a/test/runner.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- encoding: utf-8 -*- -import os -import sys - -if __name__ == '__main__': - sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) - -import test.asyncprocess as a -import test.dateandtime as td -import test.docs as tdo -import test.exceptions as te -import test.fancylogger as tf -import test.generaloption as tg -import test.missing as tm -import test.rest as trest -import test.run as trun -import test.testing as tt -import test.optcomplete as topt -import test.wrapper as wrapt -import unittest - - -from vsc.utils import fancylogger -fancylogger.logToScreen(enable=False) - -suite = unittest.TestSuite([x.suite() for x in (a, td, tg, tf, te, tm, trest, trun, tt, topt, wrapt, tdo)]) - -if __name__ == '__main__': - try: - import xmlrunner - rs = xmlrunner.XMLTestRunner(output="test-reports").run(suite) - except ImportError, err: - rs = unittest.TextTestRunner().run(suite) - - if not rs.wasSuccessful(): - sys.exit(1) From 571edb37c360585a5b51f33efb6d94e475130be5 Mon Sep 17 00:00:00 2001 From: stdweird Date: Fri, 9 Oct 2015 09:56:02 +0200 Subject: [PATCH 6/8] bump to 2.4.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c2a3072c..36c5166b 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def remove_bdist_rpm_source_file(): PACKAGE = { 'name': 'vsc-base', - 'version': '2.3.1', + 'version': '2.4.0', 'author': [sdw, jt, ag, kh], 'maintainer': [sdw, jt, ag, kh], 'packages': ['vsc', 'vsc.install', 'vsc.utils'], From 18ed55ab29ec566e811e21e66fdead7108a47aa0 Mon Sep 17 00:00:00 2001 From: stdweird Date: Fri, 9 Oct 2015 09:56:43 +0200 Subject: [PATCH 7/8] Address remark and cleanup legacy distutils-only --- lib/vsc/install/shared_setup.py | 240 ++++++++++++++++++-------------- 1 file changed, 133 insertions(+), 107 deletions(-) diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index af6ba61e..a2ffd1a5 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -25,22 +25,40 @@ import sys import re -try: - from unittest.suite import TestSuite -except ImportError: - from unittest import TestSuite - from distutils import log # also for setuptools +from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm from distutils.dir_util import remove_tree from setuptools.command.test import test as TestCommand from setuptools.command.test import ScanningLoader +from setuptools import setup +from setuptools.command.bdist_rpm import bdist_rpm +from setuptools.command.build_py import build_py +from setuptools.command.install_scripts import install_scripts +# egg_info uses sdist directly through manifest_maker +from setuptools.command.sdist import sdist +from setuptools.command.egg_info import egg_info + +from unittest import TestSuite + +# private class variables to communicate +# between VscScanningLoader and VscTestCommand +# stored in __builtin__ because the (Vsc)TestCommand.run_tests +# reloads and cleans up the modules +import __builtin__ +if not hasattr(__builtin__,'__test_filter'): + setattr(__builtin__, '__test_filter', { + 'module': None, + 'function': None, + 'allowmods': [], + }) + +# Keep this for legacy reasons, setuptools didn't used to be a requirement +has_setuptools = True # 0 : WARN (default), 1 : INFO, 2 : DEBUG log.set_verbosity(2) -has_setuptools = None - # available authors ag = ('Andy Georges', 'andy.georges@ugent.be') eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') @@ -52,11 +70,11 @@ wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') wp = ('Ward Poelmans', 'Ward.Poelmans@UGent.be') -# FIXME: do we need this here? it won;t hurt, but still ... +# Regexp used to remove suffixes from scripts when installing(/packaging) REGEXP_REMOVE_SUFFIX = re.compile(r'(\.(?:py|sh|pl))$') -# We do need all setup files to be included in the source dir if we ever want to install -# the package elsewhere. +# We do need all setup files to be included in the source dir +# if we ever want to install the package elsewhere. EXTRA_SDIST_FILES = ['setup.py'] # Put unittests under this directory @@ -65,6 +83,7 @@ URL_GH_HPCUGENT = 'https://github.com/hpcugent/%(name)s' URL_GHUGENT_HPCUGENT = 'https://github.ugent.be/hpcugent/%(name)s' + def find_extra_sdist_files(): """Looks for files to append to the FileList that is used by the egg_info.""" print "looking for extra dist files" @@ -88,65 +107,34 @@ def remove_extra_bdist_rpm_files(): """ return [] -# The following aims to import from setuptools, but if this is not available, we import the basic functionality from -# distutils instead. Note that setuptools make copies of the scripts, it does _not_ preserve symbolic links. -try: - #raise ImportError("no setuptools") # to try distutils, uncomment - from setuptools import setup - from setuptools.command.bdist_rpm import bdist_rpm - from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm - from setuptools.command.build_py import build_py - from setuptools.command.install_scripts import install_scripts - from setuptools.command.sdist import sdist - - # egg_info uses sdist directly through manifest_maker - from setuptools.command.egg_info import egg_info - - class vsc_egg_info(egg_info): - """Class to determine the set of files that should be included. - - This amounts to including the default files, as determined by setuptools, extended with the - few extra files we need to add for installation purposes. - """ - - def find_sources(self): - """Default lookup.""" - egg_info.find_sources(self) - self.filelist.extend(find_extra_sdist_files()) - # TODO: this should be in the setup.py, here we should have a placeholder, so we need not change this for every - # package we deploy - class vsc_bdist_rpm_egg_info(vsc_egg_info): - """Class to determine the source files that should be present in an (S)RPM. +class vsc_egg_info(egg_info): + """Class to determine the set of files that should be included. - All __init__.py files that augment namespace packages should be installed by the - dependent package, so we need not install it here. - """ - - def find_sources(self): - """Fins the sources as default and then drop the cruft.""" - vsc_egg_info.find_sources(self) - for f in remove_extra_bdist_rpm_files(): - print "DEBUG: removing %s from source list" % (f) - self.filelist.files.remove(f) + This amounts to including the default files, as determined by setuptools, extended with the + few extra files we need to add for installation purposes. + """ - has_setuptools = True -except ImportError as err: - log.warn("ImportError, falling back to distutils-only: %s", err) - from distutils.core import setup - from distutils.command.install_scripts import install_scripts - from distutils.command.build_py import build_py - from distutils.command.sdist import sdist - from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm - bdist_rpm = orig_bdist_rpm # mimic setuptools' bdist_rpm + def find_sources(self): + """Default lookup.""" + egg_info.find_sources(self) + self.filelist.extend(find_extra_sdist_files()) - class vsc_egg_info(object): - pass # dummy class for distutils +# TODO: this should be in the setup.py, here we should have a placeholder, so we need not change this for every +# package we deploy +class vsc_bdist_rpm_egg_info(vsc_egg_info): + """Class to determine the source files that should be present in an (S)RPM. - class vsc_bdist_rpm_egg_info(vsc_egg_info): - pass # dummy class for distutils + All __init__.py files that augment namespace packages should be installed by the + dependent package, so we need not install it here. + """ - has_setuptools = False + def find_sources(self): + """Fins the sources as default and then drop the cruft.""" + vsc_egg_info.find_sources(self) + for f in remove_extra_bdist_rpm_files(): + log.debug("removing %s from source list" % (f)) + self.filelist.files.remove(f) class vsc_install_scripts(install_scripts): @@ -187,53 +175,47 @@ def run(self): orig_bdist_rpm.run(self) -# private class variables to communicate -# between VscScanningLoader and VscTestCommand -# stored in __builtin__ because the .run_tests -# reload and cleanup in the modules -import __builtin__ -if not hasattr(__builtin__,'__test_filter'): - setattr(__builtin__, '__test_filter', { - 'module': None, - 'function': None, - 'allowmods': [], - }) - - def filter_testsuites(testsuites): """(Recursive) filtering of (suites of) tests""" - test_filter = getattr(__builtin__,'__test_filter')['function'] + test_filter = getattr(__builtin__, '__test_filter')['function'] res = type(testsuites)() - for t in testsuites: - # t is either a test or testsuite of more tests - if isinstance(t, TestSuite): - res.addTest(filter_testsuites(t)) + for ts in testsuites: + # ts is either a test or testsuite of more tests + if isinstance(ts, TestSuite): + res.addTest(filter_testsuites(ts)) else: - if re.search(test_filter, t._testMethodName): - res.addTest(t) + if re.search(test_filter, ts._testMethodName): + res.addTest(ts) return res class VscScanningLoader(ScanningLoader): + """The class to look for tests""" + + TEST_LOADER_MODULE = __name__ + def loadTestsFromModule(self, module): - """Allow for filter""" + """ + Support test module and function name based filtering + """ testsuites = ScanningLoader.loadTestsFromModule(self, module) + test_filter = getattr(__builtin__,'__test_filter') res = testsuites if test_filter['module'] is not None: - mod = module.__name__ - if mod in test_filter['allowmods']: + name = module.__name__ + if name in test_filter['allowmods']: # a parent name space pass - elif re.search(test_filter['module'], mod): + elif re.search(test_filter['module'], name): if test_filter['function'] is not None: res = filter_testsuites(testsuites) # add parents (and module itself) - pms = mod.split('.') + pms = name.split('.') for pm_idx in range(len(pms)): pm = '.'.join(pms[:pm_idx]) if not pm in test_filter['allowmods']: @@ -248,24 +230,40 @@ class VscTestCommand(TestCommand): """ The cmdclass for testing """ + + # make 2 new 'python setup.py test' options available: + # --test-filterf and --test-filertm user_options = TestCommand.user_options + [ ('test-filterf=', 'f', "Regex filter on test function names"), ('test-filterm=', 'F', "Regex filter on test (sub)modules"), ] + TEST_LOADER_CLASS = VscScanningLoader + def initialize_options(self): + """ + Add attributes for new commandline options and set test_loader + """ TestCommand.initialize_options(self) self.test_filterm = None self.test_filterf = None - self.test_loader = "vsc.install.shared_setup:VscScanningLoader" + self.test_loader = '%s:%s' % (self.TEST_LOADER_CLASS.TEST_LOADER_MODULE, self.TEST_LOADER_CLASS.__name__) + log.info("test_loader set to %s" % self.test_loader) - def run_tests(self): - getattr(__builtin__,'__test_filter').update({ - 'function': self.test_filterf, - 'module': self.test_filterm, - }) + def setup_sys_path(self): + """ + Prepare sys.path to be able to + use the modules provided by this package (assumeing they are in 'lib') + use any scripts as modules (for unittesting) + use the test modules as modules (for unittesting) + Returns a list of directories to cleanup + """ + cleanup = [] - # should be setup.py + # determine the base directory of the repository + # we will assume that the tests are called from + # a 'setup.py' like file in the basedirectory + # (but could be called anything, as long as it is in the basedir) setup_py = os.path.abspath(sys.argv[0]) log.info('run_tests from %s' % setup_py) base_dir = os.path.dirname(setup_py) @@ -275,6 +273,7 @@ def run_tests(self): lib_dir = os.path.join(base_dir, 'lib') if not os.path.exists(lib_dir): os.mkdir(lib_dir) + cleanup.append(lib_dir) test_dir = os.path.join(base_dir, DEFAULT_TEST_SUITE) if os.path.isdir(test_dir): @@ -287,22 +286,49 @@ def run_tests(self): if os.path.isdir(scripts_dir): sys.path.insert(0, scripts_dir) - # and now the ugly bits: cleanup and restore vsc namespace - # we use vsc namespace tools very early - # need to make sure they are picked up from the paths as specified above - # not to mix with installed and already loaded modules - loaded_vsc_modules = [mod for mod in sys.modules.keys() if mod == 'vsc' or mod.startswith('vsc.')] + return cleanup + + def reload_vsc_modules(self): + """ + Cleanup and restore vsc namespace becasue we use vsc namespace tools very early + So we need to make sure they are picked up from the paths as specified + in setup_sys_path, not to mix with installed and already loaded modules + """ + loaded_vsc_modules = [name for name in sys.modules.keys() if name == 'vsc' or name.startswith('vsc.')] reload_vsc_modules = [] - for mod in loaded_vsc_modules: - if hasattr(sys.modules[mod], '__file__'): + for name in loaded_vsc_modules: + if hasattr(sys.modules[name], '__file__'): # only actual modules - reload_vsc_modules.append(mod) - del(sys.modules[mod]) + reload_vsc_modules.append(name) + del(sys.modules[name]) + # reimport - for mod in reload_vsc_modules: - __import__(mod) + for name in reload_vsc_modules: + __import__(name) + + def run_tests(self): + """ + Actually run the tests, but start with + passing the filter options via __builtin__ + set sys.path + reload vsc modules + """ + getattr(__builtin__,'__test_filter').update({ + 'function': self.test_filterf, + 'module': self.test_filterm, + }) + + cleanup = self.setup_sys_path() - TestCommand.run_tests(self) + self.reload_vsc_modules() + + res = TestCommand.run_tests(self) + + # clenaup any diretcories created + for directory in cleanup: + shutil.rmtree(directory) + + return res # shared target config SHARED_TARGET = { From 7714e17fd2fb79b1ee8071c7a54eedd7cc9a9c42 Mon Sep 17 00:00:00 2001 From: stdweird Date: Fri, 9 Oct 2015 10:52:58 +0200 Subject: [PATCH 8/8] Address next round of remarks --- lib/vsc/install/shared_setup.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index a2ffd1a5..64b0c953 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -34,10 +34,10 @@ from setuptools import setup from setuptools.command.bdist_rpm import bdist_rpm from setuptools.command.build_py import build_py +from setuptools.command.egg_info import egg_info from setuptools.command.install_scripts import install_scripts # egg_info uses sdist directly through manifest_maker from setuptools.command.sdist import sdist -from setuptools.command.egg_info import egg_info from unittest import TestSuite @@ -86,14 +86,13 @@ def find_extra_sdist_files(): """Looks for files to append to the FileList that is used by the egg_info.""" - print "looking for extra dist files" + log.info("looking for extra dist files") filelist = [] for fn in EXTRA_SDIST_FILES: if os.path.isfile(fn): filelist.append(fn) else: - print "sdist add_defaults Failed to find %s" % fn - print "exiting." + log.error("sdist add_defaults Failed to find %s. Exiting." % fn) sys.exit(1) return filelist @@ -130,11 +129,11 @@ class vsc_bdist_rpm_egg_info(vsc_egg_info): """ def find_sources(self): - """Fins the sources as default and then drop the cruft.""" + """Finds the sources as default and then drop the cruft.""" vsc_egg_info.find_sources(self) - for f in remove_extra_bdist_rpm_files(): - log.debug("removing %s from source list" % (f)) - self.filelist.files.remove(f) + for fn in remove_extra_bdist_rpm_files(): + log.debug("removing %s from source list" % (fn)) + self.filelist.files.remove(fn) class vsc_install_scripts(install_scripts): @@ -231,8 +230,7 @@ class VscTestCommand(TestCommand): The cmdclass for testing """ - # make 2 new 'python setup.py test' options available: - # --test-filterf and --test-filertm + # make 2 new 'python setup.py test' options available user_options = TestCommand.user_options + [ ('test-filterf=', 'f', "Regex filter on test function names"), ('test-filterm=', 'F', "Regex filter on test (sub)modules"),