diff --git a/lib/vsc/install/shared_setup.py b/lib/vsc/install/shared_setup.py index cdd494f3..64b0c953 100644 --- a/lib/vsc/install/shared_setup.py +++ b/lib/vsc/install/shared_setup.py @@ -24,30 +24,75 @@ import shutil import sys import re + from distutils import log # also for setuptools +from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm from distutils.dir_util import remove_tree +from setuptools.command.test import test as TestCommand +from setuptools.command.test import ScanningLoader +from setuptools import setup +from setuptools.command.bdist_rpm import bdist_rpm +from setuptools.command.build_py import build_py +from setuptools.command.egg_info import egg_info +from setuptools.command.install_scripts import install_scripts +# egg_info uses sdist directly through manifest_maker +from setuptools.command.sdist import sdist + +from unittest import TestSuite + +# private class variables to communicate +# between VscScanningLoader and VscTestCommand +# stored in __builtin__ because the (Vsc)TestCommand.run_tests +# reloads and cleans up the modules +import __builtin__ +if not hasattr(__builtin__,'__test_filter'): + setattr(__builtin__, '__test_filter', { + 'module': None, + 'function': None, + 'allowmods': [], + }) + +# Keep this for legacy reasons, setuptools didn't used to be a requirement +has_setuptools = True + # 0 : WARN (default), 1 : INFO, 2 : DEBUG log.set_verbosity(2) -has_setuptools = None +# available authors +ag = ('Andy Georges', 'andy.georges@ugent.be') +eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') +jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') +kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') +kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') +lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') +sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') +wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') +wp = ('Ward Poelmans', 'Ward.Poelmans@UGent.be') +# Regexp used to remove suffixes from scripts when installing(/packaging) +REGEXP_REMOVE_SUFFIX = re.compile(r'(\.(?:py|sh|pl))$') -# We do need all setup files to be included in the source dir if we ever want to install -# the package elsewhere. +# We do need all setup files to be included in the source dir +# if we ever want to install the package elsewhere. EXTRA_SDIST_FILES = ['setup.py'] +# Put unittests under this directory +DEFAULT_TEST_SUITE = 'test' + +URL_GH_HPCUGENT = 'https://github.com/hpcugent/%(name)s' +URL_GHUGENT_HPCUGENT = 'https://github.ugent.be/hpcugent/%(name)s' + def find_extra_sdist_files(): """Looks for files to append to the FileList that is used by the egg_info.""" - print "looking for extra dist files" + log.info("looking for extra dist files") filelist = [] for fn in EXTRA_SDIST_FILES: if os.path.isfile(fn): filelist.append(fn) else: - print "sdist add_defaults Failed to find %s" % fn - print "exiting." + log.error("sdist add_defaults Failed to find %s. Exiting." % fn) sys.exit(1) return filelist @@ -61,80 +106,35 @@ def remove_extra_bdist_rpm_files(): """ return [] -# The following aims to import from setuptools, but if this is not available, we import the basic functionality from -# distutils instead. Note that setuptools make copies of the scripts, it does _not_ preserve symbolic links. -try: - #raise ImportError("no setuptools") # to try distutils, uncomment - from setuptools import setup - from setuptools.command.bdist_rpm import bdist_rpm - from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm - from setuptools.command.build_py import build_py - from setuptools.command.install_scripts import install_scripts - from setuptools.command.sdist import sdist - - # egg_info uses sdist directly through manifest_maker - from setuptools.command.egg_info import egg_info - - class vsc_egg_info(egg_info): - """Class to determine the set of files that should be included. - - This amounts to including the default files, as determined by setuptools, extended with the - few extra files we need to add for installation purposes. - """ - - def find_sources(self): - """Default lookup.""" - egg_info.find_sources(self) - self.filelist.extend(find_extra_sdist_files()) - - # TODO: this should be in the setup.py, here we should have a placeholder, so we need not change this for every - # package we deploy - class vsc_bdist_rpm_egg_info(vsc_egg_info): - """Class to determine the source files that should be present in an (S)RPM. - - All __init__.py files that augment namespace packages should be installed by the - dependent package, so we need not install it here. - """ - - def find_sources(self): - """Fins the sources as default and then drop the cruft.""" - vsc_egg_info.find_sources(self) - for f in remove_extra_bdist_rpm_files(): - print "DEBUG: removing %s from source list" % (f) - self.filelist.files.remove(f) - - has_setuptools = True -except ImportError as err: - log.warn("ImportError, falling back to distutils-only: %s", err) - from distutils.core import setup - from distutils.command.install_scripts import install_scripts - from distutils.command.build_py import build_py - from distutils.command.sdist import sdist - from distutils.command.bdist_rpm import bdist_rpm as orig_bdist_rpm - bdist_rpm = orig_bdist_rpm # mimic setuptools' bdist_rpm - class vsc_egg_info(object): - pass # dummy class for distutils +class vsc_egg_info(egg_info): + """Class to determine the set of files that should be included. - class vsc_bdist_rpm_egg_info(vsc_egg_info): - pass # dummy class for distutils + This amounts to including the default files, as determined by setuptools, extended with the + few extra files we need to add for installation purposes. + """ - has_setuptools = False + def find_sources(self): + """Default lookup.""" + egg_info.find_sources(self) + self.filelist.extend(find_extra_sdist_files()) +# TODO: this should be in the setup.py, here we should have a placeholder, so we need not change this for every +# package we deploy +class vsc_bdist_rpm_egg_info(vsc_egg_info): + """Class to determine the source files that should be present in an (S)RPM. -# available authors -ag = ('Andy Georges', 'andy.georges@ugent.be') -jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') -kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') -lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') -sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') -wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') -kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') -eh = ('Ewan Higgs', 'Ewan.Higgs@UGent.be') + All __init__.py files that augment namespace packages should be installed by the + dependent package, so we need not install it here. + """ + def find_sources(self): + """Finds the sources as default and then drop the cruft.""" + vsc_egg_info.find_sources(self) + for fn in remove_extra_bdist_rpm_files(): + log.debug("removing %s from source list" % (fn)) + self.filelist.files.remove(fn) -# FIXME: do we need this here? it won;t hurt, but still ... -REGEXP_REMOVE_SUFFIX = re.compile(r'(\.(?:py|sh|pl))$') class vsc_install_scripts(install_scripts): """Create the (fake) links for mympirun also remove .sh and .py extensions from the scripts.""" @@ -166,7 +166,7 @@ def find_package_modules (self, package, package_dir): class vsc_bdist_rpm(bdist_rpm): - """ Custom class to build the RPM, since the __inti__.py cannot be included for the packages that have namespace spread across all of the machine.""" + """Custom class to build the RPM, since the __init__.py cannot be included for the packages that have namespace spread across all of the machine.""" def run(self): log.error("vsc_bdist_rpm = %s" % (self.__dict__)) SHARED_TARGET['cmdclass']['egg_info'] = vsc_bdist_rpm_egg_info # changed to allow removal of files @@ -174,6 +174,160 @@ def run(self): orig_bdist_rpm.run(self) +def filter_testsuites(testsuites): + """(Recursive) filtering of (suites of) tests""" + test_filter = getattr(__builtin__, '__test_filter')['function'] + + res = type(testsuites)() + + for ts in testsuites: + # ts is either a test or testsuite of more tests + if isinstance(ts, TestSuite): + res.addTest(filter_testsuites(ts)) + else: + if re.search(test_filter, ts._testMethodName): + res.addTest(ts) + return res + + +class VscScanningLoader(ScanningLoader): + """The class to look for tests""" + + TEST_LOADER_MODULE = __name__ + + def loadTestsFromModule(self, module): + """ + Support test module and function name based filtering + """ + testsuites = ScanningLoader.loadTestsFromModule(self, module) + + test_filter = getattr(__builtin__,'__test_filter') + + res = testsuites + + if test_filter['module'] is not None: + name = module.__name__ + if name in test_filter['allowmods']: + # a parent name space + pass + elif re.search(test_filter['module'], name): + if test_filter['function'] is not None: + res = filter_testsuites(testsuites) + # add parents (and module itself) + pms = name.split('.') + for pm_idx in range(len(pms)): + pm = '.'.join(pms[:pm_idx]) + if not pm in test_filter['allowmods']: + test_filter['allowmods'].append(pm) + else: + res = type(testsuites)() + + return res + + +class VscTestCommand(TestCommand): + """ + The cmdclass for testing + """ + + # make 2 new 'python setup.py test' options available + user_options = TestCommand.user_options + [ + ('test-filterf=', 'f', "Regex filter on test function names"), + ('test-filterm=', 'F', "Regex filter on test (sub)modules"), + ] + + TEST_LOADER_CLASS = VscScanningLoader + + def initialize_options(self): + """ + Add attributes for new commandline options and set test_loader + """ + TestCommand.initialize_options(self) + self.test_filterm = None + self.test_filterf = None + self.test_loader = '%s:%s' % (self.TEST_LOADER_CLASS.TEST_LOADER_MODULE, self.TEST_LOADER_CLASS.__name__) + log.info("test_loader set to %s" % self.test_loader) + + def setup_sys_path(self): + """ + Prepare sys.path to be able to + use the modules provided by this package (assumeing they are in 'lib') + use any scripts as modules (for unittesting) + use the test modules as modules (for unittesting) + Returns a list of directories to cleanup + """ + cleanup = [] + + # determine the base directory of the repository + # we will assume that the tests are called from + # a 'setup.py' like file in the basedirectory + # (but could be called anything, as long as it is in the basedir) + setup_py = os.path.abspath(sys.argv[0]) + log.info('run_tests from %s' % setup_py) + base_dir = os.path.dirname(setup_py) + + # make a lib dir to trick setup.py to package this properly + # and git ignore empty dirs, so recreate it if necessary + lib_dir = os.path.join(base_dir, 'lib') + if not os.path.exists(lib_dir): + os.mkdir(lib_dir) + cleanup.append(lib_dir) + + test_dir = os.path.join(base_dir, DEFAULT_TEST_SUITE) + if os.path.isdir(test_dir): + sys.path.insert(0, test_dir) + else: + raise Exception("Can't find location of testsuite directory %s in %s" % (DEFAULT_TEST_SUITE, base_dir)) + + # make sure we can import the script as a module + scripts_dir = os.path.join(base_dir, 'bin') + if os.path.isdir(scripts_dir): + sys.path.insert(0, scripts_dir) + + return cleanup + + def reload_vsc_modules(self): + """ + Cleanup and restore vsc namespace becasue we use vsc namespace tools very early + So we need to make sure they are picked up from the paths as specified + in setup_sys_path, not to mix with installed and already loaded modules + """ + loaded_vsc_modules = [name for name in sys.modules.keys() if name == 'vsc' or name.startswith('vsc.')] + reload_vsc_modules = [] + for name in loaded_vsc_modules: + if hasattr(sys.modules[name], '__file__'): + # only actual modules + reload_vsc_modules.append(name) + del(sys.modules[name]) + + # reimport + for name in reload_vsc_modules: + __import__(name) + + def run_tests(self): + """ + Actually run the tests, but start with + passing the filter options via __builtin__ + set sys.path + reload vsc modules + """ + getattr(__builtin__,'__test_filter').update({ + 'function': self.test_filterf, + 'module': self.test_filterm, + }) + + cleanup = self.setup_sys_path() + + self.reload_vsc_modules() + + res = TestCommand.run_tests(self) + + # clenaup any diretcories created + for directory in cleanup: + shutil.rmtree(directory) + + return res + # shared target config SHARED_TARGET = { 'url': '', @@ -184,6 +338,8 @@ def run(self): "egg_info": vsc_egg_info, "bdist_rpm": vsc_bdist_rpm, }, + 'cmdclass': {'test': VscTestCommand}, + 'test_suite': DEFAULT_TEST_SUITE, } @@ -265,7 +421,7 @@ def build_setup_cfg_for_bdist_rpm(target): setup_cfg.close() -def action_target(target, setupfn=setup, extra_sdist=[]): +def action_target(target, setupfn=setup, extra_sdist=[], urltemplate=None): # EXTRA_SDIST_FILES.extend(extra_sdist) do_cleanup = True @@ -283,6 +439,11 @@ def action_target(target, setupfn=setup, extra_sdist=[]): if do_cleanup: cleanup() + if urltemplate: + target['url'] = urltemplate % target + if 'github' in urltemplate: + target['download_url'] = "%s/tarball/master" % target['url'] + build_setup_cfg_for_bdist_rpm(target) x = parse_target(target) diff --git a/setup.py b/setup.py index 0f02ad79..36c5166b 100755 --- a/setup.py +++ b/setup.py @@ -32,35 +32,36 @@ @author: Andy Georges (Ghent University) @author: Kenneth Hoste (Ghent University) """ + +# vsc-base setup.py needs vsc.install, which is currently shipped as part of vsc-base +# vsc.install doesn't require vsc-base, so we could move it to it's own repo and only +# have this hack in the setup.py of vsc.install (and set it as build_requires) +# until then... import os import sys - sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) + import vsc.install.shared_setup as shared_setup -from vsc.install.shared_setup import ag, kh, jt, sdw +from vsc.install.shared_setup import ag, kh, jt, sdw, URL_GH_HPCUGENT def remove_bdist_rpm_source_file(): """List of files to remove from the (source) RPM.""" return [] shared_setup.remove_extra_bdist_rpm_files = remove_bdist_rpm_source_file -shared_setup.SHARED_TARGET.update({ - 'url': 'https://github.com/hpcugent/vsc-base', - 'download_url': 'https://github.com/hpcugent/vsc-base', - 'zip_safe': True, -}) PACKAGE = { 'name': 'vsc-base', - 'version': '2.3.0', + 'version': '2.4.0', 'author': [sdw, jt, ag, kh], 'maintainer': [sdw, jt, ag, kh], 'packages': ['vsc', 'vsc.install', 'vsc.utils'], 'scripts': ['bin/logdaemon.py', 'bin/startlogdaemon.sh', 'bin/bdist_rpm.sh', 'bin/optcomplete.bash'], 'install_requires' : ['setuptools'], + 'zip_safe': True, } if __name__ == '__main__': - shared_setup.action_target(PACKAGE) + shared_setup.action_target(PACKAGE, urltemplate=URL_GH_HPCUGENT) diff --git a/test/runner.py b/test/runner.py deleted file mode 100644 index 2e087257..00000000 --- a/test/runner.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- encoding: utf-8 -*- -import os -import sys - -import test.asyncprocess as a -import test.dateandtime as td -import test.docs as tdo -import test.exceptions as te -import test.fancylogger as tf -import test.generaloption as tg -import test.missing as tm -import test.rest as trest -import test.run as trun -import test.testing as tt -import test.optcomplete as topt -import test.wrapper as wrapt -import unittest - - -from vsc.utils import fancylogger -fancylogger.logToScreen(enable=False) - -suite = unittest.TestSuite([x.suite() for x in (a, td, tg, tf, te, tm, trest, trun, tt, topt, wrapt, tdo)]) - -try: - import xmlrunner - rs = xmlrunner.XMLTestRunner(output="test-reports").run(suite) -except ImportError, err: - rs = unittest.TextTestRunner().run(suite) - -if not rs.wasSuccessful(): - sys.exit(1)