tasks_base.py

import datetime
import importlib
import shutil
from pathlib import Path

from invoke import run
from invoke.vendor.yaml3 import load


# trees is where individual trees are stored in directories named by the sha1 of the commits
# (A tree in git-speak is a complete snapshot of the projects' files)
trees = Path('trees')
# similar to trees, but stores virtualenvs by the sha1 of the tree that defines the requirements.txt
# by only creating a virtualenv when the requirements actually changed deployments are very fast and cost little
# disk space.
virtualenvs = trees / 'virtualenvs'
# configuration files (including the uWSGI ini's) are stored here
deployed = Path('deployed')


def nuke_path(path):
    """Recursively delete *path* (pathlib.Path)."""
    if path.exists():
        shutil.rmtree(str(path))


def create_virtualenv(tree, virtualenvs):
    """Create virtualenv and populate it with requirements."""
    last_requirements_change = get_tree_commit(tree, 'requirements.txt')
    shared_virtualenv = virtualenvs / last_requirements_change
    if not shared_virtualenv.exists():
        shared_virtualenv.mkdir(parents=True)
        print('Creatinv virtualenv {venv}'.format(venv=shared_virtualenv))
        run('virtualenv -q --python=python3 {venv}'.format(venv=shared_virtualenv))
        print('Installing requirements')
        run('{py} -m pip install -qr {reqs}'.format(py=shared_virtualenv / 'bin' / 'python3',
                                                    reqs=tree / 'requirements.txt'))
        # Make the virtualenv relocatable to allow symlinking to it
        run('virtualenv -q --python=python3 --relocatable {venv}'.format(venv=shared_virtualenv))
    virtualenv = tree / '_venv'
    virtualenv.symlink_to(shared_virtualenv.absolute())


def ref_to_sha1(commit):
    """Convert any commit specification into unique sha1 id."""
    return run('git rev-parse --default HEAD ' + commit, hide='out').stdout.strip()


def commit_is_ancestor(ancestor, descendant):
    """Return whether *ancestor* is an ancestor of (precedes) *descendant*."""
    if ancestor == descendant:
        return True
    return int(run('git rev-list --count {ancestor}..{descendant}'.format(
        ancestor=ancestor,
        descendant=descendant), hide='out').stdout) > 0


def get_tree_commit(tree, path=''):
    """Get latest (closest to HEAD) sha1 commit id that modified *path* (or any file if not *path*) from a *tree* (Path)."""
    if path:
        tree_commit = get_tree_commit(tree)
        return run('git log --format="format:%H" -1 {commit} -- {path}'.format(
            commit=tree_commit, path=path), hide='out').stdout.strip()
    return tree.name


def load_project(path):
    """Return project instance defined by project.yaml in *path*."""
    with (path / 'project.yaml').open() as file:
        project_config = load(file)
        # some.dotted.path.Class => some.dotted.path, Class
        project_module_name, project_class_name = project_config['class'].rsplit('.', maxsplit=1)
        project_module = importlib.import_module(project_module_name)
        project_class = getattr(project_module, project_class_name)
        return project_class(project_config)


class BaseUwsgiConfiguration:
    # This is a generalized version that is project and framework independent.

    # Prefix of the sha1 as generated by git-log(1)
    COMMIT_PREFIX = '# commit '

    @classmethod
    def get_commit_from_config(cls, config_path):
        with config_path.open() as file:
            for line in file:
                if line.startswith(cls.COMMIT_PREFIX):
                    return line[len(cls.COMMIT_PREFIX):].strip()

    def __init__(self, config, tree, path):
        """
        config: configuration namespace object
        tree: path to the tree we're deploying now
        path: path to the config file we're writing (.ini)
        """
        self.config = config
        self.tree = tree
        self.path = path
        self.name = self.path.name
        self.tmp_path = self.path.parent.with_suffix('.tmp')
        if self.tmp_path.is_dir():
            print('Warning: cleaning up {}'.format(self.tmp_path))
            nuke_path(self.tmp_path)
        self.tmp_path.mkdir(parents=True)

        # list of two-tuples (description, mapping-of-config-settings)
        self.sections = []

        # variables that are expanded in configuration (the usual {<keyname>} syntax)
        # useful to change touch-reload related options
        self.variables = {
            'uwsgi_ini': self.path.absolute(),
            'virtualenv': (self.path.parent / 'virtualenv').absolute(),
            'tree': (self.path.parent / 'tree').absolute(),
            'basedir': Path.cwd().absolute(),
        }

        self.make_links()

    def emplace(self):
        """Emplace configuration file."""
        self.write(self.tmp_path / self.name)
        nuke_path(self.path.parent)
        self.tmp_path.replace(self.path.parent)
        print('Created', self.path)

    def write(self, path):
        """Write config file (ini-style)."""
        with path.open('w') as file:
            print('[uwsgi]', file=file)
            self.write_info(file)

            for description, configuration in self.sections:
                print(file=file)
                print('#', description, file=file)
                for key, value in configuration.items():
                    expanded_value = str(value).format_map(self.variables)
                    print(key, '=', expanded_value, file=file)

    def write_info(self, file):
        """Write commit information to *file*."""
        print('# Generated', datetime.datetime.utcnow(), file=file)
        commit = get_tree_commit(self.tree)
        # Add log entry to ini:
        # commit 1234
        # Author: ...
        # Date:   ...
        # <commit message>
        commit_lines = run('git log -1 {commit}'.format(commit=commit), hide='out').stdout.split('\n')
        print('#', file=file)
        for line in commit_lines:
            print('#', line, file=file)

    def uwsgi_config(self):
        """Return user-specified configuration (uwsgi: section)."""
        return 'configuration from uwsgi: section', self.config.uwsgi

    def make_links(self):
        """Create symlinks from deployed/<xxx>/ directory to actual locations."""
        config_dir = self.tmp_path
        (config_dir / 'virtualenv').symlink_to((self.tree / '_venv').absolute())
        (config_dir / 'tree').symlink_to(self.tree.absolute())


class Project:
    def __init__(self, project_config):
        """*project_config* is the top-level dict parsed from the project.yaml file."""

    def make_namespace(self):
        """Return task namespace (invoke.Collection) with _additional_ (beyond deploy/checkout) tasks."""

    def uwsgi_configuration(self, ctx, tree, path):
        """Return (Base)UwsgiConfiguration object."""

    def migrate_db(self, ctx, config, from_tree, to_tree):
        """
        Migrate database of the application.

        *ctx* is the invoke context (with associated invoke.Config).
        *config* is an object as returned by Project.uwsgi_configuration.
        *from_tree* is the previously deployed tree.
        *to_tree* is the tree the database shall be migrated to.
        """