build_grimoirelab

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Jesus M. Gonzalez-Barahona <jgb@bitergia.com>
#

import argparse
import json
import logging
import os
import os.path
import shutil
import sys
import subprocess
import tempfile

description = """Create pypi pacakges for GrimoireLab.

Example:
    build_grimoirelab --build --distdir /tmp/dist

"""

build_dependencies = ['pip', 'setuptools', 'pypandoc', 'twine', 'wheel']
install_dependencies = ['pip', 'setuptools', 'wheel']

all_repos = {
    'grimoirelab': [{'name': 'grimoirelab',  'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab',
            'version_file': os.path.join('grimoirelab', '_version.py')}],
    'grimoirelab-toolkit': [{'name': 'grimoirelab-toolkit', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-toolkit',
            'version_file': os.path.join('grimoirelab_toolkit', '_version.py'),
            'test': True}],
    'perceval': [{'name': 'perceval', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-perceval',
            'version_file': os.path.join('perceval', '_version.py'),
            'check_run': 'perceval --help',
            'test': True}],
    'perceval-mozilla': [{'name': 'perceval-mozilla', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-perceval-mozilla',
            'version_file': 'setup.py',
            'check_run': 'perceval kitsune --help'}],
    'perceval-opnfv': [{'name': 'perceval-opnfv', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-perceval-opnfv',
            'version_file': 'setup.py',
            'check_run': 'perceval functest --help'}],
    'perceval-puppet': [{'name': 'perceval-puppet', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-perceval-puppet',
            'version_file': 'setup.py',
            'check_run': 'perceval puppetforge --help'}],
    'perceval-weblate': [{'name': 'perceval-weblate', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-perceval-weblate',
            'version_file': 'setup.py',
            'check_run': 'perceval weblate --help'}],
    'kingarthur': [{'name': 'kingarthur', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-kingarthur',
            'version_file': os.path.join('arthur', '_version.py'),
            'check_run': 'arthurd --help'}],
    'grimoireelk': [{'name': 'grimoire-elk', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-elk',
            'version_file': os.path.join('grimoire_elk', '_version.py'),
            'check_run': 'p2o.py --help'}],
    'sortinghat': [{'name': 'sortinghat', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-sortinghat',
            'version_file': os.path.join('sortinghat', '_version.py'),
            'check_run': 'sortinghat --help',
            'test': True}],
    'kidash': [{'name': 'kidash', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-kidash',
            'version_file': os.path.join('kidash', '_version.py'),
            'check_run': 'kidash --help'}],
    'sigils': [{'name': 'grimoirelab-panels', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-sigils',
            'version_file': os.path.join('panels', '_version.py')}],
    'sirmordred': [{'name': 'sirmordred', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-sirmordred',
            'version_file': os.path.join('sirmordred', '_version.py'),
            'check_run': 'sirmordred --help'}],
    'cereslib': [{'name': 'cereslib', 'dir': '',
            'repo_url': 'http://github.com/chaoss/grimoirelab-cereslib',
            'version_file': os.path.join('cereslib', '_version.py')}],
    'graal': [{'name': 'graal', 'dir': '',
                  'repo_url': 'http://github.com/chaoss/grimoirelab-graal',
                  'version_file': os.path.join('graal', '_version.py'),
                  'check_run': 'graal --help',
                  'test': True}]
}
"""Structure of GrimoireLab, and the specifics of the Python modules
that will come out of it.
For each git repository list of module descriptions
(package name and directory).
* Keys in the dictionary are the names in the releases file, lowecased,
and with '_' changed to '-'.
* name: name of the Python package to be produced
* repo_url: url of the git repository
* dir: directory with the pacakge in the git repository
* version_file: where to find the _version.py file, relative to the directory
"""

def parse_args ():

    parser = argparse.ArgumentParser(description = description)

    parser.add_argument("-l", "--logging", type=str, choices=["info", "debug"],
                        help="Logging level for output")
    parser.add_argument("--logfile", type=str,
                            help="Log file")

    args_actions = parser.add_argument_group("Actions", "Actions to perform")
    args_actions.add_argument("--build", action='store_true',
                            help="Build modules. "
                                + "Default action if no other is specified.")
    args_actions.add_argument("--install", action='store_true',
                            help="Install modules.")
    args_actions.add_argument("--check", action='store_true',
                            help="Check modules for some invariants.")
    args_actions.add_argument("--test", action='store_true',
                            help="Test modules.")
    args_actions.add_argument("--testinstall", action='store_true',
                            help="Test modules once installed (includes installing).")
    args_actions.add_argument("--dependencies", action='store_true',
                            help="Build a list of dependencies for this release.")

    parser.add_argument("--relfile", type=str,
                            help="GrimoireLab coordinated release file. "
                                + "If not specified, origin/master will be used "
                                + "for each module.")
    parser.add_argument("--reposfile", type=str,
                            help="File to complement repositories (JSON)")

    parser.add_argument("--modules", type=str, nargs='+',
                            help="Modules to build. Default: all")

    parser.add_argument("--reposdir", type=str,
                            help="Directory for storing git repositores. "
                                + "If not specified, use temporary directory, "
                                + "which will be wipped out when done.")
    parser.add_argument("--distdir", type=str,
                            help="Directory for storing dist packages. "
                                + "If not specified, create a random directory.")
    parser.add_argument("--confdir", type=str,
                            help="Directory for testing configuration, "
                                + "with one subdirectory per module with "
                                + "testing configuration.")
    parser.add_argument("--depfile", type=str,
                            help="File to write dependencies (eg: requirments.txt).")

    args_venv = parser.add_mutually_exclusive_group(required=False)
    args_venv.add_argument("--venv", type=str,
                            help="Python3 virtual environment for building "
                                + "packages. "
                                + "All modules needed for building will be "
                                + "installed in it in their latest version."
                                + "If not specified, create a random directory, "
                                + "which will be cleaned up when done.")
    args_venv.add_argument("--system", action='store_true',
                            help="Use Python3 system environment, instead of "
                                + "a virtual environment, for building "
                                + "packages. It should include all dependencies "
                                + "needed. Use at your own risk.")

    args_ivenv = parser.add_mutually_exclusive_group(required=False)
    args_ivenv.add_argument("--install_venv", type=str,
                            help="Python virtual environment for installing "
                                + "packages. "
                                + "All dependencies needed will be "
                                + "installed too. "
                                + "If not specified, create a random directory "
                                + "and build the virtual environment in it")
    args_ivenv.add_argument("--install_system", action='store_true',
                            help="Use Python3 system environment, instead of "
                                + "a virtual environment, for installing "
                                + "packages. "
                                + "All dependencies needed will be "
                                + "installed too. "
                                + "Ensure you have write permissions. "
                                + "Use at your own risk.")
    parser.add_argument("--install_atonce", action='store_true',
                            default=False,
                            help="Install all packages at once (True) "
                                + "or loop installing them once at a time "
                                + "(False, default).")
    parser.add_argument("--fail", action='store_true',
                            default=False,
                            help="Force the program to fail if checks "
                                + "or installation fail.")

    args = parser.parse_args()
    return args


def set_logging(args_logging, args_logfile=None):

    log_format = '%(levelname)s:%(message)s'
    if args_logging == "info":
        level = logging.INFO
    elif args_logging == "debug":
        level = logging.DEBUG
    if args_logfile:
        logging.basicConfig(format=log_format, level=level,
                            filename = args_logfile, filemode = "w")
    else:
        logging.basicConfig(format=log_format, level=level)


class Directory(object):

    def __init__(self, name=None, purpose="Generic", persistent=True):
        """Set and preparte directory.

        If name is None, create a random directory.
        If not, use name as the path name for the directory, and create it
        if it didn't exist. If a random directory is created, and
        persistent is False, remove the directory upon destruction of
        the object.

        :param       name: path name
        :param    purpose: purpose of directory (string)
        :param persistent: if False, and command line empty,
                            directory will be removed when object is destroyed

        """

        self.purpose = purpose
        if name:
            self.name = name
            self.persistent = True
            if not os.path.exists(self.name):
                os.makedirs(self.name)
        else:
            self.name = tempfile.mkdtemp()
            self.persistent = persistent
        logging.debug(self.purpose + " directory: " + self.name)
        logging.debug("  Persistent: " + str(self.persistent))

    def __del__(self):

        logging.debug("Destroying object" + self.name)
        if not self.persistent:
            logging.debug("Removing directory " + self.name)
            shutil.rmtree(self.name)


class CommandRunner (object):
    """Simple class for running commands."""

    @staticmethod
    def run_command (args, cwd='/', exit=True, env=None):
        """Run command in this virtual environment (except if self.system)

        If exit is True, and the command fails, it will exit
        printing the error when executing the command.
        Otherwise, it prints with logging.debug.
        If env is None, keep the environment variables as they are currently.
        If not, add these variables (in a dict) to the environment
        variables of the subprocess executing the command.

        :param args: command line arguments (including 0)
        :param  cwd: working directory to use
        :param exit: exit if command fails (bool)
        :param  env: environmment variables (dict), default: None
        :return    : success status (boolean) and output (string)
        """

        if env is None:
            env_vars = None
        else:
            env_vars = os.environ.copy()
            env_vars.update(env)

        result = subprocess.run(args, cwd=cwd,
                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                env=env_vars)

        if result.returncode == 0:
            success = True
            output_fn = logging.debug
            output_fn("Command successful")
        else:
            success = False
            if exit:
                output_fn = print
            else:
                output_fn = logging.debug
            output_fn("Command failed")
        output_fn('  command: ' + ' '.join(result.args))
        output = result.stdout.decode("utf-8", "backslashreplace")
        output_fn('  output: \n' + output)
        if success == False and exit:
            sys.exit(1)
        return (success, output)

    def clone_git(cls, repo, dir, commit):

        logging.debug("Cloning: " + dir)
        if not os.path.exists(dir):
            cls.run_command(['git', 'clone', repo, dir])
            logging.debug("Cloned: " + dir)
        logging.debug("Fetching: " + dir)
        cls.run_command(['git', '-C', dir, 'fetch'])
        logging.debug("Fetched: " + dir)
        cls.run_command(['git', '-C', dir, 'checkout', commit])
        logging.debug("Checked out: " + dir)

    def check_version(cls, dir, commit, version_file):

        logging.debug("Checking version in commit: " + dir)
        (success, output) = cls.run_command(['git', '-C', dir, 'diff-tree',
                '--no-commit-id', '--name-only', '-r', commit])
        check = False
        if success:
            for line in output.splitlines():
                if line == version_file:
                    check = True
        return check

class Venv (CommandRunner):
    """Class for managing virtual environments."""

    _boot = CommandRunner()

    def __init__(self, system=False, name=None, dependencies=[],
            purpose="Generic", persistent=True):
        """If name is None, create a random directory.
        If not, use name as the path name for the directory, and create it
        if it didn't exist. If a random directory is created, and
        persistent is False, remove the directory upon destruction of
        the object.

        :param       name: path name
        :param    purpose: purpose of directory (string)
        :param persistent: if False, and command line empty,
                            directory will be removed when object is destroyed

        """

        self.system = system
        if self.system:
            self.name = "System"
        else:
            self.dir = Directory(name=name, purpose=purpose, persistent=persistent)
            self.name = self.dir.name
            self._create ()
        self.update (module_names=dependencies)

    def _create(self):

        self._boot.run_command(['python3', '-m', 'venv', self.name])

    def rm(self):

        shutil.rmtree(self.name)

    def update(self, module_names, dist_dir=None, atonce=False):
        """Update virtual environment with some packages (modules).

        Use the list of modules packages for pip installing them.
        If dist_dir is specified, use it for finding the packages
        in addition to pypi. If not, use pypi only.
        All packages can be installed at once (atonce=True),
        with a single pip invocation, or one by one, with
        separate pip commands, looping through the list.

        :params module_names: list of packages names to install (strings)
        :params     dist_dir: directory with packages to install
        :params       atonce: install all packages at once or not (default: False).
        :return:              update successful (bool)
    """

        if len(module_names) == 0:
            # No packages to update, nothing to do
            return True
        common_args = ['pip3', 'install', '--upgrade']
        if dist_dir:
            common_args += ['--pre', '--find-links=' + dist_dir.name]
        if atonce:
            (success, output) = self.run_command(common_args + module_names)
        else:
            for pkg in module_names:
                (success, output) = self.run_command(common_args + [pkg])
        return success

    def print_installed(self):
        """Print list of installed packages."""

        self.run_command(['pip3', 'freeze'])

    def run_command (self, args, cwd='/', exit=True, env=None):
        """Run command in this virtual environment (except if self.system)

        If env is None, keep the environment variables as they are currently.
        If not, add these variables (in a dict) to the environment
        variables of the subprocess executing the command.

        :param args: command line arguments, including 0 (list)
        :param  cwd: working directory to use
        :param exit: exit if command fails (bool)
        :param  env: environmment variables (dict), default: None
        :return    : list with success status (boolean) and output (string)
        """

        new_args = args
        if self.system:
            new_args[0] = args[0]
        else:
            new_args[0] = os.path.join(self.name, 'bin', args[0])
        (success, output) = super().run_command(new_args, cwd=cwd,
                                                exit=exit, env=env)
        return (success, output)


class Modules(object):
    """Configuration of modules to build, install...

    Modules are pieces of Python code that are prepared to be distributed
    as sdist and wheel packages (that is they include a setup.py file, etc).
    A git repository can hold several modules in different directories.
    """

    def __init__(self, module_names=None, release=None):
        """Produce a dictionary of module descriptions, based on module names.

        Only modules in module_names will be considered (or all modules
        in the current dictionary of all repositories for GrimoireLab, if None).
        In any case, uses that dictionary to find out repos, directories,
        and the release object to find the commit to checkout.
        The resulting dictionary has as keys the module names, and as values,
        dictionaries, each with (repo, commit, dir) as keys.

        :param module_names: list of module names to consider
                            (default: None, meaining all modules considered)
        :param      release: ReleaseInfo object
        """

        self.modules = {}
        """Dictionary of module descriptions, each defined as a dictionary,
        with (repo, dir) as keys."""

        for repo, repo_modules in all_repos.items():
            logging.debug("Finding modules, considering " + repo)
            for repo_module in repo_modules:
                if (module_names is None) or (repo_module['name'] in module_names):
                    if release is None:
                        commit = 'origin/master'
                    else:
                        commit = release.get_commit(repo)
                    desc = {'repo': repo, 'commit': commit}
                    for field in repo_module:
                        if field != 'name':
                            desc[field] = repo_module[field]
                    self.modules[repo_module['name']] = desc
        logging.debug("Modules: " + str(self.modules))

    def names(self):
        """List names of modules."""

        return list(self.modules.keys())

    def descriptor(self, name):
        """Get the descriptor for a module name."""

        return self.modules[name]


class BuildingEnviron(object):
    """Environment for building modules.

    Is composed of a Modules object (modules to build),
    an Venv object (virtual environment for building modules),
    and two Directory objects (directory for cloning git repos,
    and directory for storing the distributable packages).
    """

    _runner = CommandRunner()

    def __init__(self, modules, venv, repos_dir, dist_dir, fail=False):

        self.modules = modules
        self.venv = venv
        self.repos_dir = repos_dir
        self.dist_dir = dist_dir
        self.fail = fail
        self.failures = []

    def _build_dist(self, dir):
        """Build distributable packages from a Python directory

        Build sdist and bdist_wheel packages for the directory,
        based on running the setup.py file in it.

        :param   dir: directory with the Python module
        """

        setup_file = os.path.join(dir, 'setup.py')
        if os.path.isfile(setup_file):
            self.venv.run_command(['python', 'setup.py', 'sdist',
                        '--dist-dir=' + self.dist_dir.name],
                        cwd=dir)
            self.venv.run_command(['python', 'setup.py', 'bdist_wheel',
                        '--dist-dir=' + self.dist_dir.name],
                        cwd=dir)
            return "OK"
        else:
            logging.info("Directory " + dir + " does not have a setup.py file.")
            return "Fail"

    def _test_conf(self, conf_dir, repo_dir):
        """Copy files in the test configuration directory to a repository.

        Copy all the files (if any) in the test configuration directories,
        with the same path, to the repository, which should be a clone
        of the git repo to test.

        :param conf_dir: test configuration directory
        :param repo_dir: clone of the repository
        """

        for root, dirs, files in os.walk(conf_dir):
            for file in files:
                src = os.path.join(root, file)
                dest = os.path.join(repo_dir, src[len(conf_dir)+1:])
                os.makedirs(os.path.dirname(dest), exist_ok=True)
                shutil.copyfile(src, dest)
                logging.info("Copying testing config files: " + src + " to " + dest)

    def _test_dist(self, dir, installed=False):
        """Test a Python directory (with a module)

        For testing will use packages in self.dist_dir.name, if available.

        :param       dir: directory with the Python module
        :param installed: assume module to test and dependencies are already installed
        :return:          success status (boolean) and output (string)
        """

        setup_file = os.path.join(dir, 'setup.py')
        if os.path.isfile(setup_file):
            logging.info("Running tests in {dir}. Command: python setup.py test. Env: {env}".format(
                dir=dir, env="PIP_FIND_LINKS=" + self.dist_dir.name
            ))
            if not installed:
                logging.info("Assuming not installed, running 'pip install -e .'")
                (success, output) \
                    = self.venv.run_command(['pip', 'install', '-e', '.'],
                                            cwd=dir,
                                            env={'PIP_FIND_LINKS': self.dist_dir.name})
                if not success:
                    return (success, output)
            else:
                output = ""
            (success, output_test) \
                = self.venv.run_command(['python', 'setup.py', 'test'],
                                        cwd=dir, exit=False)
            return (success, output + '\n\n' + output_test)
        else:
            logging.info("Directory " + dir + " does not have a setup.py file.")
            return (False, "Directory " + dir + " does not have a setup.py file.")

    def build_module(self, name):
        """Build module, given its name."""

        desc = self.modules.descriptor(name)
        repo_url = desc['repo_url']
        repo_dir = os.path.join(self.repos_dir.name, desc['repo'])
        pkg_dir = os.path.join(self.repos_dir.name, desc['repo'], desc['dir'])
        self._runner.clone_git(repo=repo_url, dir=repo_dir, commit=desc['commit'])
        built = self._build_dist(pkg_dir)
        print("Built module " + name + ": " + built)
        return(built)

    def test_module(self, name, conf_dir=None, installed=False):
        """Test module, given its name.

        The git repository corresponding to the module will be cloned,
        the files in the corresponding subdir of conf_dir (if any)
        will be copied, with the same path,
        to the clone of the git repo, an tests will be run.
        Returns "OK" if all tests could be run, and didn't fail,
        or "Fail" otherwise.

        :param name: name of the module to test
        :param conf_dir: name of the directory for configuration for testing
        :param installed: assume module to test and dependencies are already installed
        :return: "OK" or "Fail"
        """

        desc = self.modules.descriptor(name)
        if 'test' in desc and desc['test']:
            repo_url = desc['repo_url']
            repo_dir = os.path.join(self.repos_dir.name, desc['repo'])
            pkg_dir = os.path.join(self.repos_dir.name, desc['repo'], desc['dir'])
            if not os.path.isdir(repo_dir):
                self._runner.clone_git(repo=repo_url, dir=repo_dir, commit=desc['commit'])
            if conf_dir is not None:
                if not os.path.isdir(conf_dir):
                    print("Testing configuration directory (confdir) is not a directory:", conf_dir)
                    sys.exit(1)
                conf_dir_module = os.path.join(conf_dir, name)
                if os.path.isdir(conf_dir_module):
                    self._test_conf(conf_dir_module, pkg_dir)
            (success, output) = self._test_dist(pkg_dir, installed)
            print("Tested module " + name + ": ", end='')
            if success:
                print("OK")
            else:
                print("Fail")
                print(output)
                return("Fail")
        return("OK")

    def get_version(self, module):
        """Get the version of a module

        This is done in a virtual environment so that setup.py can run.

        :param module: module name (string)
        """

        desc = self.modules.descriptor(module)
        repo_url = desc['repo_url']
        repo_dir = os.path.join(self.repos_dir.name, desc['repo'])
        pkg_dir = os.path.join(self.repos_dir.name, desc['repo'], desc['dir'])
        if not os.path.isdir(repo_dir):
            self._runner.clone_git(repo=repo_url, dir=repo_dir, commit=desc['commit'])
        print("Getting version for {}: ".format(module), end='', flush=True)
        (success, output) = self.venv.run_command(['python', 'setup.py', '--version'],
                                                  cwd=pkg_dir, exit=False)

        version = ''
        if success:
            lines = output.splitlines()
            if len(lines) == 0:
                success = False
            else:
                version = lines[-1]
            print(version + " OK")
        if not success:
            print("Fail")
            self.failures.append("Failed to get version for {}".format(module))
        return(success, version)

    def build_all(self):
        """Build all modules."""

        some_failed = False
        for name in self.modules.names():
            built = self.build_module(name=name)
            if built == "Fail":
                some_failed = True
        if self.fail and some_failed:
            print("Some builds failed")
            sys.exit(1)

class ReleaseInfo(object):
    """For reading a coordinated release file."""

    def __init__(self, file):

        self.info = self._read(file)
        """Information about a release: dictionary with repos as keys,
        commits as values.
        """

    @staticmethod
    def _read(file):

        release = {}
        logging.debug("Reading coordinated release file")
        with open(file) as relfile:
            for line in relfile:
                line = line.strip().replace(' ','')
                parts = line.split('=')
                if len(parts) == 2:
                    repo = parts[0].lower().replace('_','-')
                    commit = parts[1].strip("'")
                    logging.debug("From release file: " + repo + " " + commit)
                    release[repo] = commit
        logging.debug("Coordinated release info: " + str(release))
        return release

    def get_commit(self, repo):
        """Get the commit corresponding to this repo in this release."""

        return self.info.get(repo)

    def get_repos(self):

        return self.info.keys()

class Checker(object):
    """Checks for a set of modules.

    :param   modules: list of modules to check
    :param repos_dir: directory for repositories (Directory)
    :param  dist_dir: directory for distribution packages (Directory)
    :param      fail: stop the program if checks fail, with errno (bool)
    """

    _check_runner = CommandRunner()

    def __init__(self, modules, repos_dir, dist_dir, fail):

        self.modules = modules
        self.repos_dir = repos_dir
        self.dist_dir = dist_dir
        self.fail = fail
        self.failures = []

    def check_version(self):
        """Check if commit changed version, for all modules."""

        print("Checking versions...")
        for module_name in self.modules.names():
            desc = self.modules.descriptor(module_name)
            if 'version_file' in desc:
                logging.debug("Checking version_file " + desc['version_file']
                    + " in " + module_name)
                repo_dir = os.path.join(self.repos_dir.name, desc['repo'])
                result = self._check_runner.check_version(dir=repo_dir,
                                        commit=desc['commit'],
                                        version_file=desc['version_file'])
                if result:
                    print("Checked version for " + module_name + ": OK")
                else:
                    print("Checked version for " + module_name + ": Fail")
                    self.failures.append("Failed version check for " + module_name)

    def check_built(self):
        """Check built modules.

        Check that built packages are usable. This means they are
        installable, and can be run.
        """

        if len(self.modules.names()) > 1:
            self.check_installable(self.modules.names())
        for module in self.modules.names():
            venv = self.check_installable([module])
            self.check_run(venv, module)

    def check_installable(self, modules):
        """Check that a list of modules are installable

        The test is done in the corresponding virtual environment.

        :param modules: list of module names (string)
        :return:        installation virtual environment (Venv)
        """

        if len(modules) > 1:
            check = "all modules at once"
        else:
            check = modules[0]
        print("Checking installable (" + check + ")...", end='', flush=True)
        install_venv = Venv(system=False,
                        dependencies=install_dependencies,
                        purpose="CheckInstallableVenv", persistent=False)
        success = install_venv.update(module_names=modules,
                                      dist_dir=self.dist_dir,
                                      atonce=True)
        if success:
            print("OK")
        else:
            print("Fail")
            self.failures.append("Failed installable (" + check + ")")
        return install_venv

    def check_run(self, venv, module):
        """Check that the run command for a module can run

        The test is done in the corresponding virtual environment.

        :param module: module name (string)
        """

        desc = self.modules.descriptor(module)
        if 'check_run' in desc:
            print("Checking run (" + module + ")...", end='', flush=True)
            command = desc['check_run'].split()
            logging.debug("Running check run for {} as {}." \
                          .format(module, command))
            (success, output) = venv.run_command(command, exit=False)
            if success:
                print("OK")
            else:
                print("Fail")
                self.failures.append("Failed check run (" + module + ")")
        else:
            logging.debug("No command to check run for {}." \
                          .format(module))

    def fail_if_failures(self):
        if self.fail and (len(self.failures) > 0):
            for failure in self.failures:
                print(failure)
            sys.exit(1)

def main():
    args = parse_args()
    if args.logging:
        set_logging(args.logging, args.logfile)

    logging.debug("Executed as: " + str(sys.argv))

    if args.relfile:
        release = ReleaseInfo(file=args.relfile)
    else:
        release = None

    if release:
        gl_rel = release.get_commit('grimoirelab-rel')
    if (not release) or (not gl_rel):
        gl_rel = 'Current master from git repos'
    print("GrimoireLab release:", gl_rel)

    if args.reposfile:
        with open(args.reposfile) as repos_file:
            repos = json.load(repos_file)
            all_repos.update(repos)

    # Make the path absolute so it works when pip is invoked from venvs
    if args.distdir:
        distdir = os.path.abspath(args.distdir)
    else:
        distdir = None

    if args.dependencies:
        if not args.depfile:
            print("--depfile option needed with --dependencies")
            sys.exit(1)

    modules = Modules(module_names=args.modules, release=release)
    dist_dir = Directory(name=distdir, purpose="Distribution",
                        persistent=True)

    # If no action is specified, let it be build
    if (not args.build) and (not args.install) and (not args.check) \
            and (not args.test) and (not args.testinstall) \
            and (not args.dependencies):
        args.build = True

    # We need repos_dir for building, checking, testing and/or dependencies
    if args.build or args.check or args.test or args.testinstall or args.dependencies:
        repos_dir = Directory(name=args.reposdir, purpose="Repos",
                                persistent=False)
    if args.build:
        print("Building...")
        venv = Venv(system=args.system, name=args.venv,
                    dependencies=build_dependencies,
                    purpose="Venv", persistent=False)
        building = BuildingEnviron(modules=modules, venv=venv,
                                   repos_dir=repos_dir, dist_dir=dist_dir,
                                   fail=args.fail)
        building.build_all()

    if args.test:
        print("Testing...")
        some_failed = False
        for module_name in modules.names():
            module = Modules(module_names=[module_name], release=release)
            venv = Venv(purpose="Test", persistent=False)
            building = BuildingEnviron(modules=module, venv=venv,
                                       repos_dir=repos_dir, dist_dir=dist_dir,
                                       fail=args.fail)
            result = building.test_module(module_name, args.confdir)
            if result == 'Fail':
                some_failed = True
        if args.fail and some_failed:
            print("Some tests failed")
            sys.exit(1)

    if args.install or args.testinstall or args.dependencies:
        install_venv = Venv(system=args.install_system, name=args.install_venv,
                            dependencies=install_dependencies,
                            purpose="InstallVenv", persistent=True)

    if args.install or args.testinstall:
        print("Installing...")
        install_venv.update(module_names=modules.names(),
                            dist_dir=dist_dir,
                            atonce=args.install_atonce)
        print("Installed modules: " + ','.join(modules.names()))

    if args.testinstall:
        print("Testing installed...")
        some_failed = False
        for module_name in modules.names():
            module = Modules(module_names=[module_name], release=release)
            building = BuildingEnviron(modules=module, venv=install_venv,
                                       repos_dir=repos_dir, dist_dir=dist_dir,
                                       fail=args.fail)
            result = building.test_module(module_name, args.confdir, installed=True)
            if result == 'Fail':
                some_failed = True
        if args.fail and some_failed:
            print("Some tests failed")
            sys.exit(1)

    if args.dependencies:
        dependencies = []
        some_failed = False

        print("Finding version numbers")
        for module_name in modules.names():
            module = Modules(module_names=[module_name], release=release)
            building = BuildingEnviron(modules=module, venv=install_venv,
                                       repos_dir=repos_dir, dist_dir=dist_dir,
                                       fail=args.fail)
            (result, version) = building.get_version(module_name)
            if result:
                dependencies.append(module_name+"=="+version)
            else:
                some_failed = True

        if args.fail and some_failed:
            print("Finding some version numbers failed")
            sys.exit(1)
        else:
            with open(args.depfile, 'w') as depfile:
                for dependency in dependencies:
                    depfile.write(dependency + '\n')


    if args.check:
        print("Checking...")
        checker = Checker(modules=modules, repos_dir=repos_dir,
                          dist_dir=dist_dir, fail=args.fail)
        checker.check_version()
        checker.check_built()
        checker.fail_if_failures()

    if args.install:
        print("Installed modules (all):")
        install_venv.print_installed()
    if args.build:
        print("Venv for building in " + venv.name)
        print("Repos for source code in " + repos_dir.name)
    print("Distribution packages in " + dist_dir.name)
    if args.install:
        print("Installed modules in " + install_venv.name)

if __name__ == "__main__":
    main()