conda-build-all

#!/usr/bin/env python
from __future__ import print_function
import re
import sys
import operator
import glob
import os
import time
import tarfile
import os.path
import argparse
from subprocess import call, check_call, check_output, CalledProcessError
from functools import reduce
from collections import defaultdict
from distutils.spawn import find_executable

BINSTAR_TOKEN = os.environ.pop('BINSTAR_TOKEN', None)

# these labels will be checked by default for existent packages.
STANDARD_LABELS = ('main',
                   'dev',
                   'rc',
                   'beta',
                   )

try:
    from conda.utils import memoized
    from conda_build.api import get_output_file_path
    import conda.api
    import conda_build.index as conda_build_index
    from conda.config import subdir as platform
    assert platform in ('osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64')
    from conda_build.metadata import MetaData
    from conda_build.config import Config as _CondaBuildConfig
    from conda.utils import url_path
    # default config instance
    CondaBuildConfig = _CondaBuildConfig()
    BUILD_DIRECTORY = CondaBuildConfig.build_prefix
except ImportError as e:
    print('ImportError encountered:')
    print(str(e))
    print('')
    print('''This script requires conda-build and anaconda-client and
must run in the root conda environment.

  $ source deactivate
  $ conda install conda-build anaconda-client''', file=sys.stderr)
    exit(1)


def main():
    p = argparse.ArgumentParser(
        'Build conda packages',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument(
        'recipe',
        action="store",
        metavar='RECIPE_PATH',
        nargs='+',
        help="path to recipe directory"
    )
    p.add_argument(
        '--check-against',
        nargs="+",
        metavar='ANACONDA_USER/label/$name',
        default=['omnia/label/%s' % label for label in STANDARD_LABELS],
    )
    p.add_argument(
        '--dry-run',
        action='store_true',
        help=('Do not perform any builds. The exit status indicates the '
              'number of build to-be-performed'),
    )
    p.add_argument('--upload',
        help='Automatically upload to binstar under this username',
    )
    p.add_argument('--force',
        action='store_true',
        dest='force',
        help='Force a package upload regardless of errors',
    )
    p.add_argument('--dev',
       action='store_true',
       dest='dev',
       help=('Push package to the dev label. This requires '
             '--force to be enabled.'),
    )
    p.add_argument(
        '--no-test',
        action='store_true',
        dest='notest',
        help="do not test the package"
    )
    p.add_argument(
        '--python',
        help="Set the Python version used by conda build",
        metavar="PYTHON_VER",
        default='27,35,36,37',

    )
    p.add_argument(
        '--numpy',
        help="Set the NumPy version used by conda build",
        metavar="NUMPY_VER",
        default='1.13,1.14,1.15',
    )
    p.add_argument(
        '-v', '--verbose',
        help='Give more output. Option is additive, and can be used up to 3 times.',
        dest='verbose',
        action='count',
    )
    args = p.parse_args()
    if args.verbose is None:
        args.verbose = 0

    if args.verbose > 2:
        print('command-line arguments:')
        print(args)

    return execute(args, p)


def toposort(data):
    """Topological sort.

    The expected input is a dictionary whose keys are items, and whose
    values are a set of the dependent items.

    The output is a genenerator over the items in topological order.
    """
    # http://code.activestate.com/recipes/578272-topological-sort/

    # Ignore self dependencies.
    for k, v in data.items():
        v.discard(k)
    # Find all items that don't depend on anything.
    extra_items_in_deps = reduce(
        set.union, data.values(), set()) - set(data.keys())
    # Add empty dependences where needed
    data.update({item: set() for item in extra_items_in_deps})
    while True:
        ordered = set(item for item, dep in data.items() if not dep)
        if not ordered:
            break
        for item in ordered:
            yield item
        data = {item: (dep - ordered) for item, dep in data.items()
                if item not in ordered}
    if data:
        raise ValueError(
            "Cyclic dependencies exist among these items"
            ":\n%s" % '\n'.join(repr(x) for x in data.items()))


def sort_recipes(recipe_paths):
    name2m = {}
    metadatas = []
    for r in recipe_paths:
        try:
            if os.path.isdir(r):
                # Skip if no meta.yaml
                meta_yaml_filename = os.path.join(r, 'meta.yaml')
                if not os.path.exists(meta_yaml_filename):
                    print("Skipping '%s' because no meta.yaml file found..." % r)
                    continue

                try:
                    m = MetaData(r)
                except (RuntimeError, IOError):
                    print('Failed to load %s' % r)
                    raise
                metadatas.append(m)
                name2m[m.get_value('package/name')] = m
        except SystemExit as e:
            # Log the error
            print("ERROR: Recipe '%s' has a fatal error that prevents processing" % r)
            print(str(e))
            # Keep going
            pass

    names = set(name2m.keys())

    graph = {}
    for m in metadatas:
        all_requirements = set(m.get_value('requirements/build', []))
        all_requirements.update(m.get_value('requirements/run', []))
        all_requirements.update(m.get_value('test/requires', []))

        our_requirements = set()
        for r in all_requirements:
            # remove any version specified in the requirements
            # (e.g. numpy >= 1.6) or something -- we just want the "numpy"
            if ' ' in r:
                r = r.split()[0]
            if r in names:
                our_requirements.add(r)

        graph[m.get_value('package/name')] = our_requirements

    return [name2m[name] for name in toposort(graph)]


@memoized
def conda_build_cmd():
    ROOTDIR = os.path.dirname(sys.executable)
    if sys.platform == 'win32':
        # find conda-build associated with the current python  interpreter
        conda_build = find_executable('conda-build', path=ROOTDIR + ';' + ROOTDIR + '\Scripts')
        assert conda_build is not None
    else:
        conda_build = find_executable('conda-build', path=ROOTDIR)
        assert conda_build is not None
    return conda_build


def get_bldpkg_path(m, python, numpy):
    cfg = _CondaBuildConfig()
    cfg.CONDA_PY = int(python.replace('.', ''))
    cfg.CONDA_NPY = int(numpy.replace('.', ''))

    m.config = cfg
    try:
        platform = m.info_index()['platform']
    except conda.exceptions.InvalidSpecError as e:
        print('Error processing package: %s' % m.name())
        raise e

    if platform is None:
        platform = 'noarch'
    cfg.platform = platform
    # need to re-parse jinja2 in light of changing config vars.
    # (esp. for skip() to work)
    m.parse_again()
    return get_output_file_path(m, config=cfg)

def list_package_contents(m, python, numpy):
    filename = get_bldpkg_path(m, python, numpy)
    if not os.path.exists(filename):
        print('ERROR: File does not exist: %s' % filename)
        return

    tarf = tarfile.open(filename, 'r:bz2')
    print('Package contents:')
    for i, name in enumerate(tarf.getnames()):
        print('  %s' % name)
        if i > 20:
            print('...')
            break

def clean_builds():
    cmd = ['conda', 'clean', '--source-cache', '--yes']
    print("Cleaning up source builds...")
    retcode = call(cmd)

def upload_to_binstar(m, python, numpy, username, max_retries=5,
                      force=False, dev=False):
    filename = get_bldpkg_path(m, python, numpy)
    if not os.path.exists(filename):
        print('ERROR: File does not exist: %s' % filename)
        return

    cmd = ['anaconda']
    if BINSTAR_TOKEN is not None:
        cmd.extend(['-t', BINSTAR_TOKEN])
    cmd.extend(['upload', '--no-progress', '-u', username, filename])
    if force:
        cmd.extend(['--force'])
    elif m.get_section('extra') and ('force_upload' in m.get_section('extra')) and (m.get_section('extra')['force_upload']=='True'):
        cmd.extend(['--force'])
    if not pre_black_listed(m) and is_prerelease(m):
        cmd.extend(['--label', 'dev'])
    elif dev:
        assert force, "Dev packages require forced uploads."
        cmd.extend(['--label', 'dev'])
    elif m.get_section('extra') and 'upload' in m.get_section('extra'):
        labels = m.get_section('extra')['upload']
        for label in labels.split(','):
            cmd.extend(['--label', label])
    else:
        cmd.extend(['--label', 'main'])

    err = None
    import subprocess
    for i in range(max_retries):
        try:
            print("Uploading package '%s' to anaconda user '%s'..." % (filename, username))
            #check_call(cmd)
            check_output(cmd, stderr=subprocess.STDOUT)
            return
        except CalledProcessError as e:
            err = e
            err.cmd = '[not shown]'
            print(' stdout:\n%s' % err.stdout)
            print(' stderr:\n%s' % err.stderr)
            if ((type(err.output) is str) and ('Distribution already exists' in err.output)) or (b'Distribution already exists' in err.output):
                # Flag in output but don't kill build for all packages
                print('*** DISTRIBUTION ALREADY EXISTS FOR PACKAGE %s' % m.name())
                return
            print("Retrying after %s seconds..." % (i+1))
            time.sleep(i+1)

    # final raise error to client
    raise err


class VersionIter(object):
    def __init__(self, pkg, metadata, versions):
        self.versions = versions
        self.metadata = metadata
        self.pkg = pkg
        reqs = self.metadata.get_value('requirements/build')
        self.reqs = [r.split(' ')[0] if ' ' in r else r for r in reqs]

    def __iter__(self):
        # build only required versions
        if self.pkg in self.reqs:
            for v in self.versions:
                yield v
        else:
            yield self.versions[0]

    def __getitem__(self, item):
        return self.versions[item]

    def __len__(self):
        if self.pkg in self.reqs:
            return len(self.versions)
        else:
            return 1


def required_builds(metadatas, pythons, numpys, channel_urls, verbose,
                    force=False):
    print('Getting package index from urls: %s' % list(channel_urls)) # DEBUG
    index = conda.api.get_index(
        channel_urls=list(channel_urls),
        prepend=False)
    index_by_pkgname = defaultdict(dict)
    for k, v in index.items():
        channel = k.channel
        pkgname = k.to_filename()
        index_by_pkgname[pkgname][channel] = v

    for m in metadatas:

        last_base_bldpkg_path = None
        python_iter = VersionIter('python', m, pythons)
        numpy_iter = VersionIter('numpy', m, numpys)
        for python in python_iter:
            for numpy in numpy_iter:

                bldpkg_path = get_bldpkg_path(m, python, numpy)
                base_bldpkg_path = os.path.basename(bldpkg_path)
                rebuild_paths = False

                # TODO: Fix this workaround more elegantly
                if python == "36" and numpy == "1.10":
                    if verbose > 1:
                        print("{pkgname} will not build on Python 3.6 and NumPy 1.10".format(pkgname=base_bldpkg_path))
                    # Check if there are more numpy versions coming:
                    if len(numpy_iter) == len(numpy_iter.versions):
                        continue
                    else:
                        # Try a higher numpy version
                        numpy = numpy_iter[1]
                        if verbose > 1:
                            print("Attempting to build {pkgname} on Python 3.6 and NumPy {numpy}".format(
                                pkgname=base_bldpkg_path, numpy=numpy))
                        rebuild_paths = True
                if rebuild_paths:
                    bldpkg_path = get_bldpkg_path(m, python, numpy)
                    base_bldpkg_path = os.path.basename(bldpkg_path)

                # Allow package recipe to force rebuild and upload in recipe.
                if m.get_section('extra') and ('force_upload' in m.get_section('extra')) and (m.get_section('extra')['force_upload']=='True'):
                    if verbose > 1:
                        print('Scheduling package due to force_upload=%s: %s' % (str(m.get_section('extra')['force_upload']), m.name()))
                        print('  full_name: %s' % base_bldpkg_path)
                        print('  python: %s' % python)
                        print('  numpy: %s' % numpy)

                    yield m, python, numpy, base_bldpkg_path

                if verbose > 2:
                    print('')
                    print('Evaluating whether to build package %s' % bldpkg_path)

                if base_bldpkg_path == last_base_bldpkg_path:
                    if verbose > 1:
                        print('base_bldpkg_path == last_base_bldpkg_path for %s' % bldpkg_path)
                    continue
                last_base_bldpkg_path = base_bldpkg_path

                if os.path.isfile(bldpkg_path):
                    if verbose > 1:
                        print('Package exists locally: %s' % base_bldpkg_path)
                    continue

                if m.skip():
                    if verbose > 1:
                        print('Skipping %s' % bldpkg_path)
                    continue

                if (base_bldpkg_path in index_by_pkgname) and not force:
                    if verbose > 2:
                        print('Package exists on anaconda.org: %s' % m.name())
                        print('  full_name:  {}'.format(base_bldpkg_path))
                        print('  channel(s): {}'.format(tuple(index_by_pkgname[base_bldpkg_path].keys())))
                        print('  md5:        {}'.format(tuple([v['md5'] for v in index_by_pkgname[base_bldpkg_path].values()])))
                    continue

                if verbose > 1:
                    print('Scheduling package: %s' % m.name())
                    print('  full_name: %s' % base_bldpkg_path)
                    print('  python: %s' % python)
                    print('  numpy: %s' % numpy)

                yield m, python, numpy, base_bldpkg_path


def build_package(m, python, numpy, args):
    cmd = [conda_build_cmd(), '-q', '--python', python, '--numpy', numpy, '--quiet']
    if args.notest:
        cmd.append('--no-test')
    # Check if there is an omnia label to build against
    if m.get_section('extra') and 'include_omnia_label' in m.get_section('extra'):
        omnia_label = m.get_section('extra')['include_omnia_label']
        if omnia_label not in STANDARD_LABELS:
            print('Cannot build package {} against label "{}", must be one of the standard labels: {}.\n'
                  'Falling back to using "main" only.'.format(m.path, omnia_label, STANDARD_LABELS))
        else:
            # Add conda forge to search AHEAD of omnia/label/LABEL otherwise the omnia channel will be higher
            # priority than the conda-forge channel, which may grab very old build which we still have on the
            # omnia channel. Its an edge case, but a critical one.
            cmd.extend(['-c', 'conda-forge', '-c', 'omnia/label/{}'.format(omnia_label)])
    cmd.append(m.path)
    retcode = call(cmd)
    list_package_contents(m, python, numpy)

    if args.upload is not None:
        if (retcode == 0):
            upload_to_binstar(m, python, numpy, username=args.upload,
                              force=args.force, dev=args.dev)
        else:
            print('Package failed to build (return code %s); will not upload.' % str(retcode))

    #clean_builds()

    sys.stdout.flush()
    return retcode


def is_prerelease(pkg_meta):
    from pkg_resources._vendor.packaging import version
    parsed = version.parse(pkg_meta.version())
    return parsed.is_prerelease


def pre_black_listed(pkg_meta):
    filename = '.pre_black_listed'
    if os.path.exists(filename):
        with open(filename) as f:
            for line in f:
                if not line.startswith('#') and pkg_meta.name() in line:
                    return True
    return False

def execute(args, p):
    pythons = re.split(',\s*|\s+', args.python)
    numpys = re.split(',\s*|\s+', args.numpy)
    channel_urls = tuple(args.check_against) if args.check_against else ()

    if args.verbose > 0:
        print('EXECUTE:')
        print('verbose: %s' % str(args.verbose))
        print('pythons: %s' % str(pythons))
        print('numpys: %s' % str(numpys))
        print('channel_urls: %s' % str(channel_urls)) # DEBUG

    args_recipe = reduce(operator.add, (glob.glob(e) for e in args.recipe))

    if args.verbose > 0:
        print('Recipes provided (in random order):')
        print(args_recipe)

    metadatas = sort_recipes(args_recipe)
    if args.verbose > 0:
        print('Considering recipes in the following order:')
        print(', '.join([os.path.basename(m.path) for m in metadatas]))
        print()
    sys.stdout.flush()

    to_build = list(required_builds(metadatas, pythons, numpys, channel_urls, verbose=args.verbose, force=args.force))

    if len(to_build) > 0:
        print('[conda-build-all] Scheduling the following builds (%s):' % platform)
    for m, python, numpy, base_bldpkg_path in to_build:
        print('  %s' % base_bldpkg_path)
    sys.stdout.flush()

    if args.dry_run:
        sys.exit(len(to_build))

    print()

    failed_builds = []
    for m, python, numpy, base_bldpkg_path in to_build:
        code = build_package(m, python, numpy, args)
        if code != 0:
            failed_builds.append(base_bldpkg_path)

    if len(failed_builds) > 0:
        print(('[conda-build-all] Error: one or more packages '
               'failed to build (%s)') % platform, file=sys.stderr)
        for b in  failed_builds:
            print('  %s' % b, file=sys.stderr)
        sys.stderr.flush()

    return len(failed_builds)


if __name__ == '__main__':
    sys.exit(main())