diff --git a/adcm_client/packer/add_to_tar.py b/adcm_client/packer/add_to_tar.py index b729dd8..2c5d711 100644 --- a/adcm_client/packer/add_to_tar.py +++ b/adcm_client/packer/add_to_tar.py @@ -16,7 +16,7 @@ def compare_helper(version, except_list): if version is None: except_list.extend([ 'spec.yaml', 'pylintrc', '.[0-9a-zA-Z]*', '*pycache*', - 'README.md', '*test*', '*requirements*', '*gz', '*md']) + 'README.md', '*test*', '*requirements*', '*.gz', '*.md']) def v_None(sub: DirEntry): for n in except_list: diff --git a/adcm_client/packer/types.py b/adcm_client/packer/types.py index 86f68d1..63dc925 100644 --- a/adcm_client/packer/types.py +++ b/adcm_client/packer/types.py @@ -1,45 +1,187 @@ import codecs import os +import random +import string +from itertools import chain -import docker import jinja2 import yaml +from docker import from_env +from docker.client import DockerClient +from docker.errors import ImageNotFound +from docker.models.containers import Container # pylint: disable=unused-import +from docker.models.images import Image + + +class NoModulesToInstall(Exception): + def __init__(self, message, errors=None): + super().__init__(message) + self.errors = errors + + +def _get_top_dirs(image: Image, prepared_image: Image, client: DockerClient) -> "list": + """Returns a list of paths to all top level folders + and files of python module that must be installed in image + :param image: image without intaled modules + :type image: Image + :param prepared_image: image with intaled modules + :type prepared_image: Image + :param d_client: docker client + :type d_client: DockerClient + :return: list of path on image fs to module files + :rtype: list + """ + # list of packages from image without installed python modules + default_module_list = _get_modules_list(image, client) + + # list of packages from image with installed installed modules + modified_module_list = _get_modules_list(prepared_image, client) + + # list of packages tham must be installed + modules = list(map( + lambda x: x.split('==')[0], + set(modified_module_list).difference(default_module_list) + )) + + modules_data = yaml.safe_load_all( + client.containers.run( + prepared_image, + 'pip show -f %s' % ' '.join(modules), + remove=True + ).decode("utf-8") + ) + return list(chain.from_iterable(map( + lambda x: [os.path.join(x['Location'], i) for i in list( + dict.fromkeys( + map( + lambda y: os.path.normpath(y).split(os.sep)[0], + x['Files'].split() + ) + )) if i not in ['..', '.']], + modules_data + ))) + + +def _get_modules_list(image: Image, client: DockerClient) -> "list": + """Run pip freeze in docker container from given image. + Returns output as a list. + + :param image: image name + :type image: Image + :param client: docker client + :type client: DockerClient + :return: list of installed python pkgs in given inamge freeze format + :rtype: list + """ + return client.containers.run( + image, '/bin/sh -c "pip freeze"', remove=True).decode("utf-8").split() + + +def _get_prepared_container(pkgs: list, image: Image, client: DockerClient) -> "Container": + """Install python pkgs to container from given image + + :param pkgs: List of python packages + :type pkgs: list + :param image: image name + :type image: Image + :param client: docker client + :type client: DockerClient + :return: container with installed python packages + :rtype: Container + """ + command = '/bin/sh -c "' + if pkgs.get('system_pkg'): + command += 'apk add ' + ' '.join(pkgs.get('system_pkg')) + ' >/dev/null ;' + command += ' pip install ' + ' '.join(pkgs.get('python_mod')) + ' >/dev/null"' + return client.containers.run(image, command, detach=True) + + +def _copy_pkgs_files(path, dirs, image: Image, volumes: dict, client: DockerClient): + """Copy list of dirs from container from given image to path. + + :param path: path where to copy + :type path: str + :param dirs: paths what need to be copied + :type dirs: list + :param image: image that contains needed paths + :type image: Image + :param volumes: volumes that must be mounted to container. + :type volumes: dict + :param client: docker client + :type client: DockerClient + """ + dirs = list(dict.fromkeys(dirs)) # filter on keys of duplicate elements + command = '/bin/sh -c "mkdir %s/pmod; cp -r %s %s/pmod ;' % (path, + ' '.join(dirs), + path) + command += ' chown -R %s %s/pmod"' % (os.getuid(), path) + client.containers.run(image, command, volumes=volumes, remove=True) + + +def _get_prepared_image(pkgs, image: Image, client: DockerClient) -> "Image": + """Install pgks to container from given image and commits container to a new image + + :return: returns image with installed required python packages + :rtype: Image + """ + container = _get_prepared_container(pkgs, image, client) + container.wait() + + prepared_image_name = [ + image.tags[0].split(':')[0], + ''.join(random.sample(string.ascii_lowercase, 5)) + ] + prepared_image = container.commit(repository=prepared_image_name[0], + tag=prepared_image_name[1]) + container.remove() + return prepared_image def python_mod_req(source_path, workspace, **kwargs): - with open(os.path.join(source_path, kwargs['requirements']), 'r') as stream: - client = docker.from_env() - image_name = "arenadata/adcm:latest" if not kwargs.get('image') else kwargs['image'] - image = client.images.pull(image_name) - data = yaml.safe_load(stream) - command = '/bin/sh -c "pip freeze"' - pmod_before = client.containers.run(image, command, remove=True).decode("utf-8").split() - - command = '/bin/sh -c "' - if data.get('system_pkg'): - command += 'apk add ' + ' '.join(data.get('system_pkg')) + ' >/dev/null ;' - if data.get('python_mod'): - command += ' pip install ' + ' '.join(data.get('python_mod')) + ' >/dev/null ;' - command += ' pip freeze"' - pmod_after = client.containers.run(image, command, remove=True).decode("utf-8").split() - - req_modules = [var for var in pmod_after if var not in pmod_before] - - command = '/bin/sh -c "' - if data.get('system_pkg'): - command += 'apk add ' + ' '.join(data.get('system_pkg')) + ' >/dev/null ;' - if data.get('python_mod'): - command += ' pip install ' + ' '.join(req_modules) + \ - ' --no-deps -t ' + source_path + '/pmod ;' - command += ' chown -R %s %s/pmod"' % (os.getuid(), source_path) - volumes = { - workspace: {'bind': workspace, 'mode': 'rw'} - } - client.containers.run(image, command, volumes=volumes, remove=True) + with open(os.path.join(source_path, kwargs['requirements']), 'r') as file: + pkgs = yaml.safe_load(file) + if pkgs.get('python_mod'): + client = from_env() + # choose image where to install python pkgs + # by default adcm:latest but i think this may be bad practice + # better to use adcm_min_version of bundle + image_name = "arenadata/adcm:latest" if not kwargs.get('image') else kwargs['image'] + image = client.images.pull(image_name) + # clean up flag + rm_prepared_image = True + + # prepared image is an image with intalled python packages + # that need to be packed in bundle + # for debug purposes there may be an prepared_image variable received from spec.yaml + if kwargs.get('prepared_image'): + try: + prepared_image = client.images.get(kwargs['prepared_image']) + rm_prepared_image = False + except ImageNotFound: + prepared_image = _get_prepared_image(pkgs, image, client) + else: + prepared_image = _get_prepared_image(pkgs, image, client) + + # list of all highlevel dirs of python pkgs that must be packed + dirs = _get_top_dirs(image, prepared_image, client) + + # volume that contains workspace + volumes = { + workspace: {'bind': workspace, 'mode': 'rw'} + } + + _copy_pkgs_files(source_path, dirs, prepared_image, volumes, client) + + if rm_prepared_image: + client.images.remove(prepared_image.id) + + else: + raise NoModulesToInstall('Can`t get python modules list to be inatalled') def splitter(*args, **kwargs): - env = jinja2.Environment(loader=jinja2.FileSystemLoader(args[0])) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(args[0]), + undefined=jinja2.StrictUndefined) for file in kwargs['files']: tmpl = env.get_template(file) with codecs.open(os.path.join(args[0], (os.path.splitext(file)[0])), 'w', 'utf-8') as f: diff --git a/setup.py b/setup.py index bad3eb7..b3d3330 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setuptools.setup( name="adcm_client", - version="2020.01.27.16", + version="2020.01.28.14", author="Anton Chevychalov", author_email="cab@arenadata.io", description="ArenaData Cluster Manager Client",