From 984200bb444529a30fe10bd686ef6119b7a0d73a Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 22 Jul 2019 20:03:32 +0200 Subject: [PATCH 01/45] 2 Bugs I found in the current installer --- askbot/deployment/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 9772633ecc..56e0f741aa 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -269,7 +269,7 @@ def _install_copy(self, copy_list, forced_overwrite=[], skip_silently=[]): print_message(' ^^^ forced overwrite!', self.verbosity) shutil.copy(src, dst) elif dst.split(os.path.sep)[-1] not in skip_silently: - print_message(f' ^^^ you already have one, please add contents of {src_file}', self.verbosity) + print_message(f' ^^^ you already have one, please add contents of {src}', self.verbosity) print_message('Done.', self.verbosity) def _install_render_with_jinja2(self, render_list, context): @@ -352,7 +352,7 @@ def _create_new_django_app(self, app_name, options): dst = os.path.join(app_dir, 'settings.py') print_message(f'Appending {options["local_settings"]} to {dst}', self.verbosity) with open(dst, 'a') as settings_file: - with open(context['local_settings'], 'r') as local_settings: + with open(options['local_settings'], 'r') as local_settings: settings_file.write('\n') settings_file.write(local_settings.read()) print_message('Done.', self.verbosity) From 0184961686b2e6b05316abdea0358f010df7b962 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 22 Jul 2019 20:04:13 +0200 Subject: [PATCH 02/45] Don't put a default for dj_database_url! Otherwise it will always overwrite the DB config in settings.py --- askbot/setup_templates/settings.py.jinja2 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/askbot/setup_templates/settings.py.jinja2 b/askbot/setup_templates/settings.py.jinja2 index 2efa5cc0ce..86c2e5f8b7 100644 --- a/askbot/setup_templates/settings.py.jinja2 +++ b/askbot/setup_templates/settings.py.jinja2 @@ -43,10 +43,10 @@ DATABASES = { {%- endif %} } -db_url = dj_database_url.config(default='sqlite:///db.data') +db_config_dict = dj_database_url.config() -if db_url: - DATABASES['default'] = db_url +if len(db_config_dict): + DATABASES['default'] = db_config_dict DATABASES['default'].update({ 'TEST': { 'CHARSET': 'utf8', # Setting the character set and collation to utf-8 }}) @@ -212,7 +212,6 @@ CACHES = { 'locmem': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'LOCATION': 'askbot', - 'TIMEOUT': 6000, } } @@ -224,7 +223,6 @@ CACHES['redis'] = { '{{ node }}', {% endfor %} ], - 'TIMEOUT': 6000, 'OPTIONS': { 'DB': '{{ cache_db }}', {% if cache_password %} @@ -248,7 +246,6 @@ CACHES['memcached'] = { '{{ node }}', {% endfor %} ], - 'TIMEOUT': 6000, } cache_select_default = 'memcached' {% else %} @@ -259,7 +256,7 @@ CACHES['default'] = CACHES[cache_select] # Chose a unique KEY_PREFIX to avoid clashes with other applications # using the same cache (e.g. a shared memcache instance). -CACHES['default'].update({'KEY_PREFIX': 'askbot',}) +CACHES['default'].update({'KEY_PREFIX': 'askbot', 'TIMEOUT': 6000}) #sets a special timeout for livesettings if you want to make them different LIVESETTINGS_CACHE_TIMEOUT = CACHES['default']['TIMEOUT'] From be49e200d4bfec2cd968ed899a2afd2c9b7aa2e4 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 22 Jul 2019 20:18:38 +0200 Subject: [PATCH 03/45] Validating setup parameters with ConfigManagers and ConfigFields * The current idea is to have a dedicated class for each install parameter * A class implements acceptable() and ask_user() * ConfigManagers check acceptable() for each parameter and call ask_user() if/while acceptable returns False * On accepting an install parameter ConfigManagers can take actions in _remember(). This is used to (slightly) alter the behaviour of the classes belonging this ConfigManager --- askbot/deployment/parameters/__init__.py | 4 + askbot/deployment/parameters/base.py | 119 +++++++++++++++++++++++ askbot/deployment/parameters/cache.py | 69 +++++++++++++ askbot/deployment/parameters/database.py | 105 ++++++++++++++++++++ 4 files changed, 297 insertions(+) create mode 100644 askbot/deployment/parameters/__init__.py create mode 100644 askbot/deployment/parameters/base.py create mode 100644 askbot/deployment/parameters/cache.py create mode 100644 askbot/deployment/parameters/database.py diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py new file mode 100644 index 0000000000..f727aa99ac --- /dev/null +++ b/askbot/deployment/parameters/__init__.py @@ -0,0 +1,4 @@ +from askbot.deployment.parameters.database import DbConfigManager +from askbot.deployment.parameters.cache import CacheConfigManager + +__all__ = [ DbConfigManager, CacheConfigManager] diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py new file mode 100644 index 0000000000..4fd325cc1c --- /dev/null +++ b/askbot/deployment/parameters/base.py @@ -0,0 +1,119 @@ +from askbot.utils import console + +class ConfigManager(object): + """ConfigManagers are used to ensure the installation can proceed. + + Each ConfigManager looks at some installation parameters, usually + grouped by the aspect of Askbot they configure. For instance there is + a ConfigManager for the database backend and another one for the cache + backend. The task of a ConfigManager is to ensure that the combination + of install parameters it analyses is sensible for an Askbot installation. + + The installer calls a ConfigManager's complete() method and passes it + its entire collection of installation parameters, a dictionary. + A ConfigManager only looks at those installation parameters which have + been register()-ed with the ConfigManager. + + For each installation parameter there is registered class, derived from + ConfigField, which can determine if a parameter's value is acceptable and + ask the user to provide a value for the parameter. + + The ConfigManager knows in which order to process its registered + installation parameters and contains all the logic determining if a user + should be asked for a(nother) value. It also remembers all values it + accepts and in the process may also modify the behaviour of the + ConfigField classes, to fit with the previously accepted values. For + instance, usually the DbConfigManager insists on credentials for accessing + the database, but if a user selects SQLite as database backend, the + DbConfigManager will *NOT* insist on credentials for accessing the + database, because SQLite does not need authentication. + """ + strings = { + 'eNoValue': 'You must specify a value for "{name}"!', + } + + def __init__(self, interactive=True): + self.interactive = interactive + self._catalog = dict() + self.keys = set() + self._managed_config = dict() + + def register(self, name, handler): + """Add the ability to handle a specific install parameter. + Parameters: + - name: the install parameter to handle + - handler: the class to handle the parameter""" + self._catalog[name] = handler + self.keys.update({name}) + + def _remember(self, name, value): + """With this method, instances remember the accepted piece of + information for a given name. Making this a method allows derived + classes to perform additional work on accepting a piece of + information.""" + self._managed_config.setdefault(name, value) + + def _complete(self, name, current_value): + """The generic procedure to ensure an installation parameter is + sensible and bug the user until a sensible value is provided. + + If this is not an interactive installation, a not acceptable() value + raises a ValueError""" + if name not in self.keys: + raise Exception + + configField = self._catalog[name] + + while not configField.acceptable(current_value): + print(f'Current value {current_value} not acceptable!') + if not self.interactive: + raise ValueError(self.strings['eNoValue'].format(name=name)) + current_value = configField.ask_user(current_value) + + # remember the piece of information we just determined acceptable() + self._remember(name, current_value) + return current_value + + def _order(self, keys): + """Gives implementations control over the order in which they process + installation parameters.""" + return keys + + def complete(self, collection): + """Main method of this :class:ConfigManager. + Consumers use this method to ensure their data in :dict:collection is + sensible for installing Askbot. + """ + contribution = dict() + keys = self.keys & set(collection.keys()) # scope to this instance + for k in self._order(keys): + v = self._complete(k, collection[k]) + contribution.setdefault(k, v) + collection.update(contribution) + +class ConfigField(object): + defaultOk = True + default = None + user_prompt = 'Please enter something' + + @classmethod + def acceptable(cls, value): + """High level sanity check for a specific value. This method is called + for an installation parameter with the value provided by the user, or + the default value, if the user didn't provide any value. There must be + a boolean response, if the installation can proceed with :var:value as + the setting for this ConfigField.""" + if value is None and cls.default is None \ + or value == cls.default: + return cls.defaultOk + return True + + @classmethod + def ask_user(cls, current): + """Prompt the user to provide a value for this installation + parameter.""" + user_prompt = cls.user_prompt + if cls.defaultOk is True: + user_prompt += ' (Just press ENTER, to use the current '\ + + f'value "{current}")' + return console.simple_dialog(user_prompt) diff --git a/askbot/deployment/parameters/cache.py b/askbot/deployment/parameters/cache.py new file mode 100644 index 0000000000..07c051132d --- /dev/null +++ b/askbot/deployment/parameters/cache.py @@ -0,0 +1,69 @@ +from askbot.utils import console +from askbot.deployment.parameters.base import ConfigField, ConfigManager + +class CacheConfigManager(ConfigManager): + """A config manager for validating setup parameters pertaining to + the cache Askbot will use.""" + def __init__(self, interactive=True): + super(CacheConfigManager, self).__init__(interactive) + self.register('cache_engine', CacheEngine) + self.register('cache_nodes', CacheNodes) + self.register('cache_db', CacheDb) + self.register('cache_password', CachePass) + + def _order(self, keys): + full_set = [ 'cache_engine', 'cache_nodes', 'cache_db', + 'cache_password' ] + return [ item for item in full_set if item in keys ] + + def _remember(self, name, value): + super(CacheConfigManager, self)._remember(name, value) + if name == 'cache_engine': + if value == '3': + self._catalog['cache_nodes'].defaultOk = True + elif value == '2': + self._catalog['cache_db'].defaultOk = False + self._catalog['cache_password'].defaultOk = False + +class CacheEngine(ConfigField): + defaultOk = False + + cache_engines = [ + ('1', 'django.core.cache.backends.memcached.MemcachedCache', 'Memcached'), + ('2', 'redis_cache.RedisCache', 'Redis'), + ('3', 'django.core.cache.backends.locmem.LocMemCache', 'LocMem'), + ] + + @classmethod + def acceptable(cls, value): + return value in [ e[0] for e in cls.cache_engines ] + + @classmethod + def ask_user(cls, current_value): + user_prompt = 'Please select cache engine:\n' + for index, name in [ (e[0], e[2]) for e in cls.cache_engines ]: + user_prompt += f'{index} - for {name}; ' + return console.choice_dialog( + user_prompt, + choices=[ e[0] for e in cls.cache_engines ] + ) + +class CacheNodes(ConfigField): + defaultOk = False + user_prompt = 'Please provide exactly one cache node in the form '\ + ':. (In order to provide multiple cache nodes, '\ + 'please use the --cache-node option multiple times when '\ + 'invoking askbot-setup.)' + + @classmethod + def ask_user(cls, current): + value = super(CacheNodes, cls).ask_user(current) + return [ value ] + +class CacheDb(ConfigField): + defaultOk = True + user_prompt = 'Please enter the cache database name to use' + +class CachePass(ConfigField): + defaultOk = True + user_prompt = 'Please enter the shared secret for accessing the cache' diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py new file mode 100644 index 0000000000..f666e30347 --- /dev/null +++ b/askbot/deployment/parameters/database.py @@ -0,0 +1,105 @@ +from askbot.utils import console +import os +from askbot.deployment import path_utils + +from askbot.deployment.parameters.base import ConfigField, ConfigManager + +class DbConfigManager(ConfigManager): + """A config manager for validating setup parameters pertaining to + the database Askbot will use.""" + def __init__(self, interactive=True): + super(DbConfigManager, self).__init__(interactive) + self.register('database_engine', DbEngine) + self.register('database_name', DbName) + self.register('database_user', DbUser) + self.register('database_password', DbPass) + self.register('database_host', DbHost) + self.register('database_port', DbPort) + + def _order(self, keys): + full_set = [ 'database_engine', 'database_name', 'database_user', + 'database_password', 'database_host', 'database_port' ] + return [ item for item in full_set if item in keys ] + + def _remember(self, name, value): + super(DbConfigManager, self)._remember(name, value) + if name == 'database_engine': + self._catalog['database_name'].db_type = value + self._catalog['database_name'].set_user_prompt() + if value == '2': + self._catalog['database_user'].defaultOk = True + self._catalog['database_password'].defaultOk = True + +class DbEngine(ConfigField): + defaultOk = False + + database_engines = [ + ('1', 'postgresql_psycopg2', 'PostgreSQL'), + ('2', 'sqlite3', 'SQLite'), + ('3', 'mysql', 'MySQL'), + ('4', 'oracle', 'Oracle'), + ] + + @classmethod + def acceptable(cls, value): + return value in [ e[0] for e in cls.database_engines ] + + @classmethod + def ask_user(cls, current_value): + user_prompt = 'Please select database engine:\n' + for index, name in [ (e[0], e[2]) for e in cls.database_engines ]: + user_prompt += f'{index} - for {name}; ' + return console.choice_dialog( + user_prompt, + choices=[ e[0] for e in cls.database_engines ] + ) + +class DbName(ConfigField): + defaultOk = False + db_type = 1 + + @classmethod + def set_user_prompt(cls): + if cls.db_type == '2': + cls.user_prompt = 'Please enter database file name' + else: + cls.user_prompt = 'Please enter database name' + + @classmethod + def acceptable(cls, value): + if value is None: + return False + if cls.db_type != '2': + return len(value.split(' ')) < 2 + if os.path.isfile(value): + message = 'file %s exists, use it anyway?' % value + if console.get_yes_or_no(message) == 'yes': + return True + elif os.path.isdir(value): + print('%s is a directory, choose another name' % value) + elif value in path_utils.FILES_TO_CREATE: + print('name %s cannot be used for the database name' % value) + elif value == path_utils.LOG_DIR_NAME: + print('name %s cannot be used for the database name' % value) + else: + return True + return False + +class DbUser(ConfigField): + defaultOk = False + user_prompt = 'Please enter the username for accessing the database' + +class DbPass(ConfigField): + defaultOk = False + user_prompt = 'Please enter the password for the database user' + +# As it stands, we will never prompt the user to provide DbHost nor DbPort. +# One would have to add a class method acceptable() which can actually fail to +# create use cases where Askbot install may prompt the user. +class DbHost(ConfigField): + defaultOk = True + user_prompt = 'Please enter the database hostname' + +class DbPort(ConfigField): + defaultOk = True + user_prompt = 'Please enter the database port' From 5b5f16a743fc72524ac2d8bb61537d15c76dd22e Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 24 Jul 2019 22:23:43 +0200 Subject: [PATCH 04/45] Code refinement and testcases * Added askbot/tests/test_installer.py which currently only tests the askbot.deployment.parameters module * The 10 added tests throw various sets of install parameters at the parameters module w.r.t the database and cache settings. * The parameters module currently only checks the database_engine and cache_engine option for sensible values. For all other parameters, it only ensures that the user did provide parameters at all, not if they are sensible or not. --- askbot/deployment/parameters/__init__.py | 2 +- askbot/deployment/parameters/base.py | 40 ++- askbot/deployment/parameters/cache.py | 67 ++--- askbot/deployment/parameters/database.py | 107 ++++---- askbot/tests/test_installer.py | 307 +++++++++++++++++++++++ 5 files changed, 429 insertions(+), 94 deletions(-) create mode 100644 askbot/tests/test_installer.py diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index f727aa99ac..09175a083e 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -1,4 +1,4 @@ from askbot.deployment.parameters.database import DbConfigManager from askbot.deployment.parameters.cache import CacheConfigManager -__all__ = [ DbConfigManager, CacheConfigManager] +__all__ = [ 'DbConfigManager', 'CacheConfigManager'] diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index 4fd325cc1c..b937223e34 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -1,6 +1,14 @@ from askbot.utils import console -class ConfigManager(object): +class ObjectWithOutput(object): + def __init__(self, verbosity=1): + self.verbosity = verbosity + + def print(self, message, verbosity=1): + if verbosity <= self.verbosity: + print(message) + +class ConfigManager(ObjectWithOutput): """ConfigManagers are used to ensure the installation can proceed. Each ConfigManager looks at some installation parameters, usually @@ -32,7 +40,8 @@ class ConfigManager(object): 'eNoValue': 'You must specify a value for "{name}"!', } - def __init__(self, interactive=True): + def __init__(self, interactive=True, verbosity=1): + super(ConfigManager, self).__init__(verbosity=verbosity) self.interactive = interactive self._catalog = dict() self.keys = set() @@ -45,6 +54,7 @@ def register(self, name, handler): - handler: the class to handle the parameter""" self._catalog[name] = handler self.keys.update({name}) + handler.verbosity = self.verbosity def _remember(self, name, value): """With this method, instances remember the accepted piece of @@ -65,7 +75,7 @@ def _complete(self, name, current_value): configField = self._catalog[name] while not configField.acceptable(current_value): - print(f'Current value {current_value} not acceptable!') + self.print(f'Current value {current_value} not acceptable!', 2) if not self.interactive: raise ValueError(self.strings['eNoValue'].format(name=name)) current_value = configField.ask_user(current_value) @@ -91,29 +101,33 @@ def complete(self, collection): contribution.setdefault(k, v) collection.update(contribution) -class ConfigField(object): +class ConfigField(ObjectWithOutput): defaultOk = True default = None user_prompt = 'Please enter something' - @classmethod - def acceptable(cls, value): + def __init__(self, defaultOk=None, default=None, user_prompt=None, verbosity=1): + super(ConfigField, self).__init__(verbosity=verbosity) + self.defaultOk = self.__class__.defaultOk if defaultOk is None else defaultOk + self.default = self.__class__.default if default is None else default + self.user_prompt = self.__class__.user_prompt if user_prompt is None else user_prompt + + def acceptable(self, value): """High level sanity check for a specific value. This method is called for an installation parameter with the value provided by the user, or the default value, if the user didn't provide any value. There must be a boolean response, if the installation can proceed with :var:value as the setting for this ConfigField.""" - if value is None and cls.default is None \ - or value == cls.default: - return cls.defaultOk + #self.print(f'This is {cls.__name__}.acceptable({value}) {cls.defaultOk}', 2) + if value is None and self.default is None or value == self.default: + return self.defaultOk return True - @classmethod - def ask_user(cls, current): + def ask_user(self, current): """Prompt the user to provide a value for this installation parameter.""" - user_prompt = cls.user_prompt - if cls.defaultOk is True: + user_prompt = self.user_prompt + if self.defaultOk is True: user_prompt += ' (Just press ENTER, to use the current '\ + f'value "{current}")' return console.simple_dialog(user_prompt) diff --git a/askbot/deployment/parameters/cache.py b/askbot/deployment/parameters/cache.py index 07c051132d..d31dd6c8f4 100644 --- a/askbot/deployment/parameters/cache.py +++ b/askbot/deployment/parameters/cache.py @@ -4,12 +4,20 @@ class CacheConfigManager(ConfigManager): """A config manager for validating setup parameters pertaining to the cache Askbot will use.""" - def __init__(self, interactive=True): - super(CacheConfigManager, self).__init__(interactive) - self.register('cache_engine', CacheEngine) - self.register('cache_nodes', CacheNodes) - self.register('cache_db', CacheDb) - self.register('cache_password', CachePass) + def __init__(self, interactive=True, verbosity=1): + super(CacheConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) + db = ConfigField( + defaultOk = True, + user_prompt = 'Please enter the cache database name to use', + ) + password = ConfigField( + defaultOk = True, + user_prompt = 'Please enter the shared secret for accessing the cache', + ) + self.register('cache_engine', CacheEngine()) + self.register('cache_nodes', CacheNodes()) + self.register('cache_db', db) + self.register('cache_password', password) def _order(self, keys): full_set = [ 'cache_engine', 'cache_nodes', 'cache_db', @@ -19,9 +27,9 @@ def _order(self, keys): def _remember(self, name, value): super(CacheConfigManager, self)._remember(name, value) if name == 'cache_engine': - if value == '3': + if value == 3: self._catalog['cache_nodes'].defaultOk = True - elif value == '2': + elif value == 2: self._catalog['cache_db'].defaultOk = False self._catalog['cache_password'].defaultOk = False @@ -29,25 +37,33 @@ class CacheEngine(ConfigField): defaultOk = False cache_engines = [ - ('1', 'django.core.cache.backends.memcached.MemcachedCache', 'Memcached'), - ('2', 'redis_cache.RedisCache', 'Redis'), - ('3', 'django.core.cache.backends.locmem.LocMemCache', 'LocMem'), + (1, 'django.core.cache.backends.memcached.MemcachedCache', 'Memcached'), + (2, 'redis_cache.RedisCache', 'Redis'), + (3, 'django.core.cache.backends.locmem.LocMemCache', 'LocMem'), ] - @classmethod - def acceptable(cls, value): - return value in [ e[0] for e in cls.cache_engines ] + def acceptable(self, value): + return value in [e[0] for e in self.cache_engines] - @classmethod - def ask_user(cls, current_value): + def ask_user(self, current_value, depth=0): user_prompt = 'Please select cache engine:\n' - for index, name in [ (e[0], e[2]) for e in cls.cache_engines ]: + for index, name in [(e[0], e[2]) for e in self.cache_engines]: user_prompt += f'{index} - for {name}; ' - return console.choice_dialog( + user_input = console.choice_dialog( user_prompt, - choices=[ e[0] for e in cls.cache_engines ] + choices=[e[0] for e in self.cache_engines] ) + try: + user_input = int(user_input) + except Exception as e: + if depth > 7: + raise e + self.print(e) + user_input = self.ask_user(user_input, depth + 1) + + return user_input + class CacheNodes(ConfigField): defaultOk = False user_prompt = 'Please provide exactly one cache node in the form '\ @@ -55,15 +71,6 @@ class CacheNodes(ConfigField): 'please use the --cache-node option multiple times when '\ 'invoking askbot-setup.)' - @classmethod - def ask_user(cls, current): - value = super(CacheNodes, cls).ask_user(current) + def ask_user(self, current): + value = super(CacheNodes, self).ask_user(current) return [ value ] - -class CacheDb(ConfigField): - defaultOk = True - user_prompt = 'Please enter the cache database name to use' - -class CachePass(ConfigField): - defaultOk = True - user_prompt = 'Please enter the shared secret for accessing the cache' diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index f666e30347..8dbf3b4858 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -7,14 +7,33 @@ class DbConfigManager(ConfigManager): """A config manager for validating setup parameters pertaining to the database Askbot will use.""" - def __init__(self, interactive=True): - super(DbConfigManager, self).__init__(interactive) - self.register('database_engine', DbEngine) - self.register('database_name', DbName) - self.register('database_user', DbUser) - self.register('database_password', DbPass) - self.register('database_host', DbHost) - self.register('database_port', DbPort) + def __init__(self, interactive=True, verbosity=1): + super(DbConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) + engine = DbEngine() + name = DbName() + username = ConfigField( + defaultOk=False, + user_prompt='Please enter the username for accessing the database', + ) + password = ConfigField( + defaultOk=False, + user_prompt='Please enter the password for accessing the database', + ) + host = ConfigField( + defaultOk=True, + user_prompt='Please enter the database hostname', + ) + port = ConfigField( + defaultOk=True, + user_prompt='Please enter the database port', + ) + + self.register('database_engine', engine) + self.register('database_name', name) + self.register('database_user', username) + self.register('database_password', password) + self.register('database_host', host) + self.register('database_port', port) def _order(self, keys): full_set = [ 'database_engine', 'database_name', 'database_user', @@ -26,7 +45,7 @@ def _remember(self, name, value): if name == 'database_engine': self._catalog['database_name'].db_type = value self._catalog['database_name'].set_user_prompt() - if value == '2': + if value == 2: self._catalog['database_user'].defaultOk = True self._catalog['database_password'].defaultOk = True @@ -34,72 +53,60 @@ class DbEngine(ConfigField): defaultOk = False database_engines = [ - ('1', 'postgresql_psycopg2', 'PostgreSQL'), - ('2', 'sqlite3', 'SQLite'), - ('3', 'mysql', 'MySQL'), - ('4', 'oracle', 'Oracle'), + (1, 'postgresql_psycopg2', 'PostgreSQL'), + (2, 'sqlite3', 'SQLite'), + (3, 'mysql', 'MySQL'), + (4, 'oracle', 'Oracle'), ] - @classmethod - def acceptable(cls, value): - return value in [ e[0] for e in cls.database_engines ] + def acceptable(self, value): + return value in [e[0] for e in self.database_engines] - @classmethod - def ask_user(cls, current_value): + def ask_user(self, current_value, depth=0): user_prompt = 'Please select database engine:\n' - for index, name in [ (e[0], e[2]) for e in cls.database_engines ]: + for index, name in [(e[0], e[2]) for e in self.database_engines]: user_prompt += f'{index} - for {name}; ' - return console.choice_dialog( + user_input = console.choice_dialog( user_prompt, - choices=[ e[0] for e in cls.database_engines ] + choices=[e[0] for e in self.database_engines] ) + try: + user_input = int(user_input) + except Exception as e: + if depth > 7: + raise e + self.print(e) + user_input = self.ask_user(user_input, depth + 1) + + return user_input + class DbName(ConfigField): defaultOk = False db_type = 1 - @classmethod - def set_user_prompt(cls): - if cls.db_type == '2': - cls.user_prompt = 'Please enter database file name' + def set_user_prompt(self): + if self.db_type == 2: + self.user_prompt = 'Please enter database file name' else: - cls.user_prompt = 'Please enter database name' + self.user_prompt = 'Please enter database name' - @classmethod - def acceptable(cls, value): + def acceptable(self, value): if value is None: return False - if cls.db_type != '2': + if self.db_type != 2: return len(value.split(' ')) < 2 if os.path.isfile(value): message = 'file %s exists, use it anyway?' % value if console.get_yes_or_no(message) == 'yes': return True elif os.path.isdir(value): - print('%s is a directory, choose another name' % value) + self.print('%s is a directory, choose another name' % value) elif value in path_utils.FILES_TO_CREATE: - print('name %s cannot be used for the database name' % value) + self.print('name %s cannot be used for the database name' % value) elif value == path_utils.LOG_DIR_NAME: - print('name %s cannot be used for the database name' % value) + self.print('name %s cannot be used for the database name' % value) else: return True return False -class DbUser(ConfigField): - defaultOk = False - user_prompt = 'Please enter the username for accessing the database' - -class DbPass(ConfigField): - defaultOk = False - user_prompt = 'Please enter the password for the database user' - -# As it stands, we will never prompt the user to provide DbHost nor DbPort. -# One would have to add a class method acceptable() which can actually fail to -# create use cases where Askbot install may prompt the user. -class DbHost(ConfigField): - defaultOk = True - user_prompt = 'Please enter the database hostname' - -class DbPort(ConfigField): - defaultOk = True - user_prompt = 'Please enter the database port' diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py new file mode 100644 index 0000000000..d9a790ac50 --- /dev/null +++ b/askbot/tests/test_installer.py @@ -0,0 +1,307 @@ +from askbot.tests.utils import AskbotTestCase +from askbot.deployment import AskbotSetup + +from askbot.deployment.parameters import * + +## Database config related tests +class DbConfigManagerTest(AskbotTestCase): + + def setUp(self): + self.installer = AskbotSetup() + self.parser = self.installer.parser + + def test_get_options(self): + default_opts = self.parser.parse_args([]) + default_dict = vars(default_opts) + + def test_db_configmanager(self): + manager = DbConfigManager(interactive=False, verbosity=0) + new_empty = lambda:dict([(k,None) for k in manager.keys]) + + parameters = new_empty() # includes ALL database parameters + self.assertGreater(len(parameters), 0) + + engines = manager._catalog['database_engine'].database_engines + self.assertGreater(len(engines), 0) + + # with interactive=False ConfigManagers (currently) raise an exception + # if a required parameter is not set or not acceptable + # the ConfigManager must trip over missing/empty database settings + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + self.assertIs(type(e), ValueError) + + +class DatabaseEngineTest(AskbotTestCase): + + def setUp(self): + self.installer = AskbotSetup() + self.parser = self.installer.parser + + def test_database_engine(self): + manager = DbConfigManager(interactive=False, verbosity=0) + new_empty = lambda: dict([(k, None) for k in manager.keys]) + + + # DbConfigManager is supposed to test database_engine first + # here: engine is NOT acceptable and name is NOT acceptable + parameters = new_empty() # includes ALL database parameters + try: + manager.complete(parameters) + except ValueError as e: + self.assertIn('database_engine', str(e)) + + # With a database_engine set, users must provide a database_name + # here: engine is acceptable and name is NOT acceptable + engines = manager._catalog['database_engine'].database_engines + parameters = {'database_engine': None, 'database_name': None} + caught_exceptions = 0 + for db_type in [e[0] for e in engines]: + parameters['database_engine'] = db_type + try: + manager.complete(parameters) + except ValueError as ve: + caught_exceptions += 1 + self.assertIn('database_name', str(ve)) + self.assertEquals(caught_exceptions, len(engines)) + + # here: engine is not acceptable and name is acceptable + parameters = {'database_engine': None, 'database_name': 'acceptable_value'} + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + self.assertIn('database_engine', str(e)) + + # here: engine is acceptable and name is acceptable + acceptable_engine = manager._catalog['database_engine'].database_engines[0][0] + parameters = {'database_engine': acceptable_engine, 'database_name': 'acceptable_value'} + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + self.assertIsNone(e) + + # at the moment, the parameter parse does not have special code for + # mysql and oracle, so we do not provide dedicated tests for them + def test_database_postgres(self): + manager = DbConfigManager(interactive=False, verbosity=0) + new_empty = lambda: dict([(k, None) for k in manager.keys]) + parameters = new_empty() + parameters['database_engine'] = 1 + + acceptable_answers = { + 'database_name': 'testDB', + 'database_user': 'askbot', + 'database_password': 'd34db33f', + } + expected_issues = acceptable_answers.keys() + met_issues = set() + for i in expected_issues: + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + matches = [issue for issue in expected_issues if issue in str(e)] + self.assertEqual(len(matches), 1, str(e)) + self.assertIs(type(e), ValueError) + + issue = matches[0] + cnt_old = len(met_issues) + met_issues.update({issue}) + cnt_new = len(met_issues) + self.assertNotEqual(cnt_new, cnt_old) + parameters[issue] = acceptable_answers[issue] + self.assertEqual(len(expected_issues), len(met_issues)) + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + self.assertIsNone(e) + + def test_database_sqlite(self): + manager = DbConfigManager(interactive=False, verbosity=0) + new_empty = lambda: dict([(k, None) for k in manager.keys]) + parameters = new_empty() + parameters['database_engine'] = 2 + + acceptable_answers = { + 'database_name': 'testDB', + } + expected_issues = acceptable_answers.keys() + met_issues = set() + for i in expected_issues: + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + matches = [issue for issue in expected_issues if issue in str(e)] + self.assertEqual(len(matches), 1, str(e)) + self.assertIs(type(e), ValueError) + + issue = matches[0] + cnt_old = len(met_issues) + met_issues.update({issue}) + cnt_new = len(met_issues) + self.assertNotEqual(cnt_new, cnt_old) + parameters[issue] = acceptable_answers[issue] + self.assertEqual(len(expected_issues), len(met_issues)) + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + self.assertIsNone(e) + +## Cache config related tests +class CacheEngineTest(AskbotTestCase): + + def setUp(self): + self.installer = AskbotSetup() + self.parser = self.installer.parser + + @staticmethod + def _setUpTest(): + manager = CacheConfigManager(interactive=False, verbosity=0) + engines = manager._catalog['cache_engine'].cache_engines + new_empty = lambda: dict([(k, None) for k in manager.keys]) + return manager, engines, new_empty + + @staticmethod + def run_complete(manager, parameters): + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + return e + + def test_cache_configmanager(self): + manager, engines, new_empty = self._setUpTest() + + parameters = new_empty() # includes ALL cache parameters + self.assertGreater(len(parameters), 0) + self.assertGreater(len(engines), 0) + + # with interactive=False ConfigManagers (currently) raise an exception + # if a required parameter is not set or not acceptable + # the ConfigManager must trip over missing/empty cache settings + parameters = {'cache_engine': None} + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + + def test_cache_engine(self): + manager, engines, new_empty = self._setUpTest() + + # CacheConfigManager is supposed to test cache_engine first + # here: engine is NOT acceptable and nodes is NOT acceptable + parameters = new_empty() # includes ALL cache parameters + + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + self.assertIn('cache_engine', str(e)) + + # here: engine is acceptable and nodes is NOT acceptable + parameters = {'cache_engine': None, 'cache_nodes': None} + caught_exceptions = 0 + for db_type in [e[0] for e in engines if e[2] != 'LocMem']: + parameters['cache_engine'] = db_type + e = self.run_complete(manager, parameters) + if type(e) is ValueError: + caught_exceptions += 1 + self.assertIn('cache_nodes', str(e)) + self.assertEquals(caught_exceptions, len(engines) - 1) + + # here: engine is not acceptable and nodes is acceptable + parameters = {'cache_engine': None, 'cache_nodes': ['127.0.0.1:1234']} + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + self.assertIn('cache_engine', str(e)) + + # here: engine is acceptable and nodes is acceptable + not_locmem = engines[0] + if not_locmem[2] == 'LocMem': + not_locmem = engines[1] + acceptable_engine = not_locmem[0] + parameters = {'cache_engine': acceptable_engine, 'cache_nodes': ['127.0.0.1:1234']} + e = self.run_complete(manager, parameters) + self.assertIsNone(e) + + def test_locmem_engine(self): + manager, engines, new_empty = self._setUpTest() + + # when the engine is locmem, the default must be acceptable for all + # other cache_* parameters, because locmem does not require any of them + locmem = [ e for e in engines if e[2] == 'LocMem'][0] + parameters = new_empty() + parameters.update({'cache_engine': locmem[0]}) + e = self.run_complete(manager, parameters) + self.assertIsNone(e) + + def test_cache_redis(self): + manager, engines, new_empty = self._setUpTest() + + redis = [e for e in engines if e[2] == 'Redis'][0] + parameters = new_empty() + parameters['cache_engine'] = redis[0] + + acceptable_answers = { + 'cache_nodes': '127.0.0.1:6379', + 'cache_db': 1, + 'cache_password': 'd34db33f', + } + expected_issues = acceptable_answers.keys() + met_issues = set() + for i in expected_issues: + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + + matches = [issue for issue in expected_issues if issue in str(e)] + self.assertEqual(len(matches), 1, str(e)) + issue = matches[0] + cnt_old = len(met_issues) + met_issues.update({issue}) + cnt_new = len(met_issues) + self.assertNotEqual(cnt_new, cnt_old) + + parameters[issue] = acceptable_answers[issue] + self.assertEqual(len(expected_issues), len(met_issues)) + e = self.run_complete(manager, parameters) + self.assertIsNone(e) + + def test_cache_memcached(self): + manager, engines, new_empty = self._setUpTest() + + memcached = [e for e in engines if e[2] == 'Memcached'][0] + parameters = new_empty() + parameters['cache_engine'] = memcached[0] + + acceptable_answers = { + 'cache_nodes': '127.0.0.1:11211', + } + expected_issues = acceptable_answers.keys() + met_issues = set() + for i in expected_issues: + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + + matches = [issue for issue in expected_issues if issue in str(e)] + self.assertEqual(len(matches), 1, str(e)) + issue = matches[0] + cnt_old = len(met_issues) + met_issues.update({issue}) + cnt_new = len(met_issues) + self.assertNotEqual(cnt_new, cnt_old) + + parameters[issue] = acceptable_answers[issue] + self.assertEqual(len(expected_issues), len(met_issues)) + e = self.run_complete(manager, parameters) + self.assertIsNone(e) From 0904b836f3a51e84b1242625e5180f97a3f695c3 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sun, 28 Jul 2019 22:16:51 +0200 Subject: [PATCH 05/45] Fixed types and added ConfigManagerCollection * Even though the default values are of type int, the parser arguments for cache_engine and database_engine are retrieved as strings. Added casts to int to fix this issue * First integration efforts indicated it makes sense to collect all ConfigManagers to make everything accessible via a single object the installer can use --- askbot/deployment/parameters/__init__.py | 11 +++++++++++ askbot/deployment/parameters/base.py | 2 +- askbot/deployment/parameters/cache.py | 10 +++++----- askbot/deployment/parameters/database.py | 15 ++++++++++----- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index 09175a083e..dac6f3c62e 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -1,4 +1,15 @@ from askbot.deployment.parameters.database import DbConfigManager from askbot.deployment.parameters.cache import CacheConfigManager +class ConfigManagerCollection(object): + def __init__(self, interactive=False, verbosity=0): + self.verbosity = verbosity + self.interactive = interactive + self.database = DbConfigManager(interactive=interactive, verbosity=verbosity) + self.cache = CacheConfigManager(interactive=interactive, verbosity=verbosity) + + def complete(self, *args, **kwargs): + self.database.complete(*args, **kwargs) + self.cache.complete(*args, **kwargs) + __all__ = [ 'DbConfigManager', 'CacheConfigManager'] diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index b937223e34..34d0e0e68c 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -70,7 +70,7 @@ def _complete(self, name, current_value): If this is not an interactive installation, a not acceptable() value raises a ValueError""" if name not in self.keys: - raise Exception + raise KeyError configField = self._catalog[name] diff --git a/askbot/deployment/parameters/cache.py b/askbot/deployment/parameters/cache.py index d31dd6c8f4..879670bcca 100644 --- a/askbot/deployment/parameters/cache.py +++ b/askbot/deployment/parameters/cache.py @@ -25,11 +25,11 @@ def _order(self, keys): return [ item for item in full_set if item in keys ] def _remember(self, name, value): - super(CacheConfigManager, self)._remember(name, value) + super(CacheConfigManager, self)._remember(name, int(value)) if name == 'cache_engine': - if value == 3: + if int(value) == 3: self._catalog['cache_nodes'].defaultOk = True - elif value == 2: + elif int(value) == 2: self._catalog['cache_db'].defaultOk = False self._catalog['cache_password'].defaultOk = False @@ -43,7 +43,7 @@ class CacheEngine(ConfigField): ] def acceptable(self, value): - return value in [e[0] for e in self.cache_engines] + return int(value) in [e[0] for e in self.cache_engines] def ask_user(self, current_value, depth=0): user_prompt = 'Please select cache engine:\n' @@ -51,7 +51,7 @@ def ask_user(self, current_value, depth=0): user_prompt += f'{index} - for {name}; ' user_input = console.choice_dialog( user_prompt, - choices=[e[0] for e in self.cache_engines] + choices=[str(e[0]) for e in self.cache_engines] ) try: diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index 8dbf3b4858..4b60299669 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -41,11 +41,11 @@ def _order(self, keys): return [ item for item in full_set if item in keys ] def _remember(self, name, value): - super(DbConfigManager, self)._remember(name, value) + super(DbConfigManager, self)._remember(name, int(value)) if name == 'database_engine': - self._catalog['database_name'].db_type = value + self._catalog['database_name'].db_type = int(value) self._catalog['database_name'].set_user_prompt() - if value == 2: + if int(value) == 2: self._catalog['database_user'].defaultOk = True self._catalog['database_password'].defaultOk = True @@ -60,7 +60,12 @@ class DbEngine(ConfigField): ] def acceptable(self, value): - return value in [e[0] for e in self.database_engines] + self.print(f'DbEngine.complete called with {value} of type {type(value)}', 2) + try: + return int(value) in [e[0] for e in self.database_engines] + except: + pass + return False def ask_user(self, current_value, depth=0): user_prompt = 'Please select database engine:\n' @@ -68,7 +73,7 @@ def ask_user(self, current_value, depth=0): user_prompt += f'{index} - for {name}; ' user_input = console.choice_dialog( user_prompt, - choices=[e[0] for e in self.database_engines] + choices=[str(e[0]) for e in self.database_engines] ) try: From 8258e7b2cf5931f82316bccc1465110f6aafc819 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sun, 28 Jul 2019 22:20:12 +0200 Subject: [PATCH 06/45] Bugfix for the old installer * fixed import statement for SettingsTemplate * refactored an open() close() instance with "with open(..." --- askbot/deployment/path_utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/askbot/deployment/path_utils.py b/askbot/deployment/path_utils.py index 879fc908a4..b8e813cae7 100644 --- a/askbot/deployment/path_utils.py +++ b/askbot/deployment/path_utils.py @@ -15,7 +15,7 @@ from askbot.deployment import messages from askbot.utils import console -from askbot.deployment.template_loader import DeploymentTemplate +from askbot.deployment.template_loader import DeploymentTemplate as SettingsTemplate FILES_TO_CREATE = ('__init__.py', 'manage.py', 'urls.py', 'django.wsgi', 'celery_app.py') @@ -98,12 +98,10 @@ def has_existing_django_project(directory): if file_name.endswith(os.path.sep + 'manage.py'): #a hack allowing to install into the distro directory continue - py_file = open(file_name) - for line in py_file: - if IMPORT_RE1.match(line) or IMPORT_RE2.match(line): - py_file.close() - return True - py_file.close() + with open(file_name, 'r') as py_file: + for line in py_file: + if IMPORT_RE1.match(line) or IMPORT_RE2.match(line): + return True return False From 01c0a2a8e9281ab6ff7043bed7cddb9b6c7361c8 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sun, 28 Jul 2019 22:23:51 +0200 Subject: [PATCH 07/45] Shuffled existing argparser arguments and added 3 * The arguments added through _add_settings_args and _add_setup_args were not sorted properly. They now are * When running askbot-setup -h, the output depends on the ordering of the code. To make the installer more usable I pushed the db and cache related arguments to the end of the list and put the '--' names before the '-' names. * Three new arguments to askbot-setup_ --dry-run: Print options_dict and exit jsut before the deployment would start --use_defaults: No function yet. There is supposed to be an explicit dict for an Askbot default installation. This dict will be forced into options, if this argument is provided --no-input No function yet. The idea is to make the installer fail instead of asking the user for missing input, if this argument is provided. --- askbot/deployment/__init__.py | 140 ++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 49 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 56e0f741aa..42f0a1f400 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -31,77 +31,100 @@ def __init__(self): self._add_arguments() def _add_arguments(self): - self._add_db_args() - self._add_cache_args() self._add_setup_args() self._add_settings_args() + self._add_db_args() + self._add_cache_args() + def _add_settings_args(self): """Misc parameters for rendering settings.py Adds --logfile-name + --append-settings --no-secret-key - --create-project """ + self.parser.add_argument( "--logfile-name", dest="logfile_name", default='askbot.log', help="name of the askbot logfile." - ) + ) self.parser.add_argument( - "--no-secret-key", - dest="no_secret_key", - action='store_true', - default=False, - help="Don't generate a secret key. (not recommended)" - ) + "--append-settings", + dest="local_settings", + default='', + help="Extra settings file to append custom settings" + ) self.parser.add_argument( - '--create-project', - dest='create_project', - action='store', - default='django', - help='Deploy a new Django project (default)' + "--no-secret-key", + dest="no_secret_key", + action='store_true', + help="Don't generate a secret key. (not recommended)" ) def _add_setup_args(self): """Control the behaviour of this setup procedure Adds + --create - project + --dir-name, -n + --verbose, -v - --append-settings --force - --dir-name, -n """ + self.parser.add_argument( - "-v", "--verbose", + '--create-project', + dest='create_project', + action='store', + default='django', + help='Deploy a new Django project (default)' + ) + + self.parser.add_argument( + "--dir-name", "-n", + dest = "dir_name", + default = None, + help = "Directory where you want to install." + ) + + self.parser.add_argument( + "--verbose", "-v", dest = "verbosity", default = 1, help = "verbosity level available values 0, 1, 2." - ) + ) self.parser.add_argument( - "--append-settings", - dest = "local_settings", - default = '', - help = "Extra settings file to append custom settings" - ) + "--force", + dest="force", + action='store_true', + help="Force overwrite settings.py file" + ) self.parser.add_argument( - "--force", - dest="force", - action='store_true', - default=False, - help = "Force overwrite settings.py file" - ) + "--dry-run", + dest = "dry_run", + action='store_true', + help="Dump parameters and do not install askbot after input validation." + ) self.parser.add_argument( - "-n", "--dir-name", - dest = "dir_name", - default = None, - help = "Directory where you want to install." - ) + "--use-defaults", + dest="use_defaults", + action='store_true', + help="Use Askbot defaults where applicable. Defaults will be overwritten by commandline arguments." + ) + + self.parser.add_argument( + "--no-input", + dest="interactive", + action='store_false', + help="The installer will fail instead of asking for missing values." + ) def _add_cache_args(self): """Cache settings @@ -109,8 +132,8 @@ def _add_cache_args(self): self.parser.add_argument('--cache-engine', dest='cache_engine', action='store', - default='locmem', - help='Select with Django cache backend to use. ' + default=None, + help='Select which Django cache backend to use. ' ) self.parser.add_argument('--cache-node', @@ -145,47 +168,47 @@ def _add_db_args(self): --db-port """ self.parser.add_argument( - '-e', '--db-engine', + '--db-engine', '-e', dest='database_engine', action='store', choices=DATABASE_ENGINE_CHOICES, - default=2, - help='Database engine, type 1 for postgresql, 2 for sqlite, 3 for mysql' + default=None, + help='Database engine, type 1 for PostgreSQL, 2 for SQLite, 3 for MySQL, 4 for Oracle' ) self.parser.add_argument( - "-d", "--db-name", + "--db-name", "-d", dest = "database_name", default = None, - help = "The database name" + help = "The database name Askbot will use" ) self.parser.add_argument( - "-u", "--db-user", + "--db-user", "-u", dest = "database_user", default = None, - help = "The database user" + help = "The username Askbot uses to connect to the database" ) self.parser.add_argument( - "-p", "--db-password", + "--db-password", "-p", dest = "database_password", default = None, - help = "the database password" + help = "The password Askbot uses to connect to the database" ) self.parser.add_argument( "--db-host", dest = "database_host", default = None, - help = "the database host" + help = "The database host" ) self.parser.add_argument( "--db-port", dest = "database_port", default = None, - help = "the database host" + help = "The database port" ) def _set_verbosity(self, options): @@ -213,7 +236,7 @@ def __call__(self): # this is the main part of the original askbot_setup() # the destination directory directory = path_utils.clean_directory(options.dir_name) while directory is None: - directory = path_utils.get_install_directory(force=options.get('force')) # i.e. ask the user + directory = path_utils.get_install_directory(force=options.force) # i.e. ask the user options.dir_name = directory if options.database_engine not in DATABASE_ENGINE_CHOICES: @@ -224,6 +247,10 @@ def __call__(self): # this is the main part of the original askbot_setup() ) options_dict = vars(options) + + #from copy import deepcopy + #options_dict_2 = deepcopy(options_dict) + if options.force is False: options_dict = collect_missing_options(options_dict) @@ -236,6 +263,21 @@ def __call__(self): # this is the main part of the original askbot_setup() database_engine = database_engine_codes[options.database_engine] options_dict['database_engine'] = database_engine + #from askbot.deployment.parameters import ConfigManagerCollection + + #cm = ConfigManagerCollection(interactive=True, verbosity=2) + #cm.complete(options_dict_2) + + + + + if options_dict['dry_run'] == True: + from pprint import pprint + pprint(options_dict) + #pprint(options_dict_2) + pprint(self.__dict__) + raise KeyboardInterrupt + self.deploy_askbot(options_dict) if database_engine == 'postgresql_psycopg2': From ccfde5c755f408983db142b23215a45ad0d6ae45 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 29 Jul 2019 09:39:35 +0200 Subject: [PATCH 08/45] Add force attribute to base class --- askbot/deployment/parameters/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index 34d0e0e68c..94aa62842b 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -1,8 +1,9 @@ from askbot.utils import console class ObjectWithOutput(object): - def __init__(self, verbosity=1): + def __init__(self, verbosity=1, force=False): self.verbosity = verbosity + self.force = force def print(self, message, verbosity=1): if verbosity <= self.verbosity: From 33018aa3673b837a976c725b808e4eec3166a773 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 29 Jul 2019 09:39:55 +0200 Subject: [PATCH 09/45] Fixed int conversion of *_engine paramters --- askbot/deployment/parameters/cache.py | 10 ++++++---- askbot/deployment/parameters/database.py | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/askbot/deployment/parameters/cache.py b/askbot/deployment/parameters/cache.py index 879670bcca..86d8ecd1e0 100644 --- a/askbot/deployment/parameters/cache.py +++ b/askbot/deployment/parameters/cache.py @@ -25,11 +25,13 @@ def _order(self, keys): return [ item for item in full_set if item in keys ] def _remember(self, name, value): - super(CacheConfigManager, self)._remember(name, int(value)) if name == 'cache_engine': - if int(value) == 3: + value = int(value) + super(CacheConfigManager, self)._remember(name, value) + if name == 'cache_engine': + if value == 3: self._catalog['cache_nodes'].defaultOk = True - elif int(value) == 2: + elif value == 2: self._catalog['cache_db'].defaultOk = False self._catalog['cache_password'].defaultOk = False @@ -43,7 +45,7 @@ class CacheEngine(ConfigField): ] def acceptable(self, value): - return int(value) in [e[0] for e in self.cache_engines] + return value in [e[0] for e in self.cache_engines] def ask_user(self, current_value, depth=0): user_prompt = 'Please select cache engine:\n' diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index 4b60299669..10a5a73a6d 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -41,11 +41,13 @@ def _order(self, keys): return [ item for item in full_set if item in keys ] def _remember(self, name, value): - super(DbConfigManager, self)._remember(name, int(value)) if name == 'database_engine': - self._catalog['database_name'].db_type = int(value) + value = int(value) + super(DbConfigManager, self)._remember(name, value) + if name == 'database_engine': + self._catalog['database_name'].db_type = value self._catalog['database_name'].set_user_prompt() - if int(value) == 2: + if value == 2: self._catalog['database_user'].defaultOk = True self._catalog['database_password'].defaultOk = True @@ -62,7 +64,7 @@ class DbEngine(ConfigField): def acceptable(self, value): self.print(f'DbEngine.complete called with {value} of type {type(value)}', 2) try: - return int(value) in [e[0] for e in self.database_engines] + return value in [e[0] for e in self.database_engines] except: pass return False From 63712edeb77d705aba182fa8b7e03ae01e1a919f Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 29 Jul 2019 09:40:50 +0200 Subject: [PATCH 10/45] Fix docstrings to allign code description with code --- askbot/deployment/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 42f0a1f400..90b3176192 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -71,9 +71,10 @@ def _add_setup_args(self): Adds --create - project --dir-name, -n - --verbose, -v --force + --dry-run + --use-defaults """ self.parser.add_argument( @@ -128,6 +129,11 @@ def _add_setup_args(self): def _add_cache_args(self): """Cache settings + Adds + --cache-engine + --cache-node + --cache-db + --cache-password """ self.parser.add_argument('--cache-engine', dest='cache_engine', From 19ab8fe240214161f36f2068c88eb5c137c4a51d Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 29 Jul 2019 09:41:38 +0200 Subject: [PATCH 11/45] Introduced FilesystemConfigManager * no integration yet --- askbot/deployment/parameters/filesystem.py | 86 ++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 askbot/deployment/parameters/filesystem.py diff --git a/askbot/deployment/parameters/filesystem.py b/askbot/deployment/parameters/filesystem.py new file mode 100644 index 0000000000..5e9814a35a --- /dev/null +++ b/askbot/deployment/parameters/filesystem.py @@ -0,0 +1,86 @@ +from askbot.utils import console +from askbot.deployment.parameters.base import ConfigField, ConfigManager +from askbot.deployment import messages +from askbot.deployment.path_utils import has_existing_django_project + +import os.path +import re +from importlib.machinery import PathFinder +import tempfile + +class FilesystemConfigManager(ConfigManager): + """A config manager for validating setup parameters pertaining to + files and directories Askbot will use.""" + + def __init__(self, interactive=True, verbosity=1): + super(FilesystemConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) + logfile = ConfigField( + defaultOk=True, + user_prompt="Please enter the name for Askbot's logfile.", + ) + self.register('dir_name', DirName()) + self.register('logfile_name', logfile) + + + def _order(self, keys): + full_set = ['dir_name', 'logfile_name'] + return [item for item in full_set if item in keys] + +class DirNameError(Exception): + """There is something about the chosen install dir we don't like.""" + +class DirName(ConfigField): + defaultOk = False + + def _check_django_name_restrictions(self, directory): + dir_name = os.path.basename(directory) + if not re.match(r'[_a-zA-Z][\w-]*$', dir_name): + raise DirNameError("""\nDirectory %s is not acceptable for a Django + project. Please use lower case characters, numbers and underscore. + The first character cannot be a number.\n""" % os.path.basename(directory)) + + def _check_module_name_collision(self, directory): + dir_name = os.path.basename(directory) + finder = PathFinder.find_spec(dir_name,os.path.dirname(directory)) + if finder is not None: + raise DirNameError(messages.format_msg_bad_dir_name(directory)) + + def _check_is_file(self, directory): + directory = os.path.normpath(directory) + directory = os.path.abspath(directory) + if os.path.isfile(directory): + raise DirNameError(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) + + def _check_can_create_write_path(self, directory): + if not os.path.exists(directory): + self._check_can_create_write_path(os.path.dirname(directory)) + else: + try: + with tempfile.NamedTemporaryFile(dir=directory) as f: + f.write("Hello World!") + except: + raise DirNameError(messages.format_msg_dir_not_writable(directory)) + + def _check_nested_django_projects(self, directory): + if has_existing_django_project(directory): + raise DirNameError(messages.format_msg_dir_unclean_django(directory)) + if len(os.path.split(directory)[1].strip()) > 0: + self._check_nested_django_projects(os.path.dirname(directory)) + + def _check_forced_overwrite(self, directory): + if has_existing_django_project(directory) and self.force is False: + raise DirNameError(messages.CANNOT_OVERWRITE_DJANGO_PROJECT % \ + {'directory': directory}) + + def acceptable(self, value): + try: + self._check_django_name_restrictions(value) + self._check_module_name_collision(value) + self._check_is_file(value) + self._check_can_create_write_path(value) + self._check_nested_django_projects(os.path.dirname(value)) + self._check_forced_overwrite(value) + except DirNameError as error: + self.print(error) + return False + return True From 76adba668a498d75e5ee2ed95d012b7f5fa2a475 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 30 Jul 2019 22:25:56 +0200 Subject: [PATCH 12/45] Add --app-name as parameter to overwrite 'askbot_app' --- askbot/deployment/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 90b3176192..d4f6333683 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -88,8 +88,15 @@ def _add_setup_args(self): self.parser.add_argument( "--dir-name", "-n", dest = "dir_name", - default = None, - help = "Directory where you want to install." + default = '', + help = "Directory where you want to install the Django project." + ) + + self.parser.add_argument( + "--app-name", + dest="app_name", + default='askbot_app', + help="Django app name (subdir) for this Askbot deployment in the target Django project." ) self.parser.add_argument( From 125693150cf11ad4ca3264d4713f18fc8013fabe Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 30 Jul 2019 22:27:09 +0200 Subject: [PATCH 13/45] Add convenience function to access registered handlers --- askbot/deployment/parameters/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index 94aa62842b..9ebc9e3a9b 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -57,6 +57,11 @@ def register(self, name, handler): self.keys.update({name}) handler.verbosity = self.verbosity + def configField(self, name): + if name not in self.keys: + raise KeyError(f'{self.__class__.__name__}: No handler for {name} registered.') + return self._catalog[name] + def _remember(self, name, value): """With this method, instances remember the accepted piece of information for a given name. Making this a method allows derived From f2e433e3aa674387a542dc6c15dd00d313865b47 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 30 Jul 2019 23:19:32 +0200 Subject: [PATCH 14/45] Improved FilesystemConfigManager (WiP) * lots of changes to existing code * also manages app_name now --- askbot/deployment/parameters/filesystem.py | 100 ++++++++++++++++----- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/askbot/deployment/parameters/filesystem.py b/askbot/deployment/parameters/filesystem.py index 5e9814a35a..7155d405d2 100644 --- a/askbot/deployment/parameters/filesystem.py +++ b/askbot/deployment/parameters/filesystem.py @@ -1,13 +1,15 @@ from askbot.utils import console -from askbot.deployment.parameters.base import ConfigField, ConfigManager from askbot.deployment import messages +from askbot.deployment.parameters.base import ConfigField, ConfigManager from askbot.deployment.path_utils import has_existing_django_project +from importlib.util import find_spec import os.path import re -from importlib.machinery import PathFinder import tempfile +DEBUG_VERBOSITY = 2 + class FilesystemConfigManager(ConfigManager): """A config manager for validating setup parameters pertaining to files and directories Askbot will use.""" @@ -18,69 +20,125 @@ def __init__(self, interactive=True, verbosity=1): defaultOk=True, user_prompt="Please enter the name for Askbot's logfile.", ) - self.register('dir_name', DirName()) + self.register('dir_name', ProjectDirName()) + self.register('app_name', AppDirName()) self.register('logfile_name', logfile) - def _order(self, keys): - full_set = ['dir_name', 'logfile_name'] + full_set = ['dir_name', 'app_name', 'logfile_name'] return [item for item in full_set if item in keys] class DirNameError(Exception): """There is something about the chosen install dir we don't like.""" -class DirName(ConfigField): +class RestrictionsError(DirNameError): + pass + +class NameCollisionError(DirNameError): + pass + +class IsFileError(DirNameError): + pass + +class CreateWriteError(DirNameError): + pass + +class NestedProjectsError(DirNameError): + pass + +class OverwriteError(DirNameError): + pass + +class BaseDirName(ConfigField): defaultOk = False def _check_django_name_restrictions(self, directory): dir_name = os.path.basename(directory) - if not re.match(r'[_a-zA-Z][\w-]*$', dir_name): - raise DirNameError("""\nDirectory %s is not acceptable for a Django + if re.match(r'[_a-zA-Z][\w-]*$', dir_name) is None: + raise RestrictionsError("""\nDirectory %s is not acceptable for a Django project. Please use lower case characters, numbers and underscore. The first character cannot be a number.\n""" % os.path.basename(directory)) def _check_module_name_collision(self, directory): dir_name = os.path.basename(directory) - finder = PathFinder.find_spec(dir_name,os.path.dirname(directory)) - if finder is not None: - raise DirNameError(messages.format_msg_bad_dir_name(directory)) + spec = find_spec(dir_name,os.path.dirname(directory)) + if spec is not None: + raise NameCollisionError(messages.format_msg_bad_dir_name(directory)) def _check_is_file(self, directory): directory = os.path.normpath(directory) directory = os.path.abspath(directory) if os.path.isfile(directory): - raise DirNameError(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) + raise IsFileError(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) def _check_can_create_write_path(self, directory): + self.print(f'_check_can_create_write_path({directory})', DEBUG_VERBOSITY) if not os.path.exists(directory): self._check_can_create_write_path(os.path.dirname(directory)) else: try: with tempfile.NamedTemporaryFile(dir=directory) as f: - f.write("Hello World!") + f.write(b"Hello World!") except: - raise DirNameError(messages.format_msg_dir_not_writable(directory)) + raise CreateWriteError(messages.format_msg_dir_not_writable(directory)) def _check_nested_django_projects(self, directory): if has_existing_django_project(directory): - raise DirNameError(messages.format_msg_dir_unclean_django(directory)) + raise NestedProjectsError(messages.format_msg_dir_unclean_django(directory)) if len(os.path.split(directory)[1].strip()) > 0: self._check_nested_django_projects(os.path.dirname(directory)) def _check_forced_overwrite(self, directory): if has_existing_django_project(directory) and self.force is False: - raise DirNameError(messages.CANNOT_OVERWRITE_DJANGO_PROJECT % \ + raise OverwriteError(messages.CANNOT_OVERWRITE_DJANGO_PROJECT % \ {'directory': directory}) +class ProjectDirName(BaseDirName): + def acceptable(self, value): + self.print(f'Got "{value}" of type "{type(value)}".', DEBUG_VERBOSITY) + try: + self._check_django_name_restrictions(value) + self._check_module_name_collision(value) + path_to_value = os.path.abspath(value) + self._check_is_file(path_to_value) + self._check_can_create_write_path(path_to_value) + self._check_nested_django_projects(os.path.dirname(path_to_value)) + self._check_forced_overwrite(path_to_value) + except DirNameError as error: + self.print(f'{error.__class__.__name__}:', DEBUG_VERBOSITY) + self.print(error, 1) + return False + return True + + def ask_user(self, current_value): + self.user_prompt = messages.WHERE_TO_DEPLOY + user_input = os.path.abspath( + super(ProjectDirName, self).ask_user(current_value)) + + should_create_new = console.choice_dialog( + messages.format_msg_create(user_input), + choices=['yes', 'no'], + invalid_phrase=messages.INVALID_INPUT + ) + + return None if should_create_new == 'no' else user_input + +class AppDirName(BaseDirName): + defaultOk = True, + default = 'askbot_app', + user_prompt = "Please enter a Django App name for this Askbot deployment." + def acceptable(self, value): + self.print(f'Got "{value}" of type "{type(value)}".', DEBUG_VERBOSITY) try: self._check_django_name_restrictions(value) self._check_module_name_collision(value) - self._check_is_file(value) - self._check_can_create_write_path(value) - self._check_nested_django_projects(os.path.dirname(value)) - self._check_forced_overwrite(value) + if os.path.sep in value: + raise DirNameError(f'The App name must be a single valid name without any path information, not {value}.') + path_to_value = os.path.abspath(value) + self._check_is_file(path_to_value) except DirNameError as error: - self.print(error) + self.print(f'{error.__class__.__name__}:', DEBUG_VERBOSITY) + self.print(error, 1) return False return True From fa35388850507964be75a0ad363b4a94a64a4dbc Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 30 Jul 2019 23:20:42 +0200 Subject: [PATCH 15/45] Added tests for FilesystemConfigManager --- askbot/deployment/parameters/__init__.py | 3 +- askbot/tests/test_installer.py | 116 ++++++++++++++++++++++- 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index dac6f3c62e..0249e50119 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -1,5 +1,6 @@ from askbot.deployment.parameters.database import DbConfigManager from askbot.deployment.parameters.cache import CacheConfigManager +from askbot.deployment.parameters.filesystem import FilesystemConfigManager class ConfigManagerCollection(object): def __init__(self, interactive=False, verbosity=0): @@ -12,4 +13,4 @@ def complete(self, *args, **kwargs): self.database.complete(*args, **kwargs) self.cache.complete(*args, **kwargs) -__all__ = [ 'DbConfigManager', 'CacheConfigManager'] +__all__ = [ 'DbConfigManager', 'CacheConfigManager', 'FilesystemConfigManager'] diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index d9a790ac50..f892b62085 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -1,8 +1,19 @@ from askbot.tests.utils import AskbotTestCase from askbot.deployment import AskbotSetup - from askbot.deployment.parameters import * +from unittest.mock import patch + +class MockInput: + def __init__(self, *args): + self.return_values = iter(args) + + def __call__(self, *args): + value = next(self.return_values) + #print(f'MockInput called with >>>{args}<<<; answering >>>{value}<<<', file=sys.stderr) + return value + + ## Database config related tests class DbConfigManagerTest(AskbotTestCase): @@ -305,3 +316,106 @@ def test_cache_memcached(self): self.assertEqual(len(expected_issues), len(met_issues)) e = self.run_complete(manager, parameters) self.assertIsNone(e) + +class FilesystemTests(AskbotTestCase): + + def setUp(self): + self.installer = AskbotSetup() + self.parser = self.installer.parser + + @staticmethod + def _setUpTest(): + manager = FilesystemConfigManager(interactive=False, verbosity=0) + new_empty = lambda: dict([(k, None) for k in manager.keys]) + return manager, new_empty + + @staticmethod + def run_complete(manager, parameters): + e = None + try: + manager.complete(parameters) + except ValueError as ve: + e = ve + return e + + def test_filesystem_configmanager(self): + manager, new_empty = self._setUpTest() + + parameters = new_empty() # includes ALL cache parameters + self.assertGreater(len(parameters), 0) + + # with interactive=False ConfigManagers (currently) raise an exception + # if a required parameter is not set or not acceptable + # the ConfigManager must trip over missing/empty cache settings + parameters = {'dir_name': ''} + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + + def test_project_dir(self): + manager, new_empty = self._setUpTest() + + failing_dir_names = [ + '', # empty string violates name restriction + 'os', # name of a module in PYTHONPATH causes a name collision + 'askbot', # name of a module in PYTHONPATH causes a name collision + '/bin/bash', # is a file + '/root', # cannot write there + '/usr/local/lib/python3.x/dist-packages/some-package/module/submodule/subsubmodule', # practice recursion + ] + for name in failing_dir_names: + parameters = {'dir_name': name} + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError) + + valid_dir_names = [ + 'validDeployment', + '/tmp/validDeployment', + ] + for name in valid_dir_names: + parameters = {'dir_name': name} + e = self.run_complete(manager, parameters) + self.assertIsNone(e) + + def test_app_name(self): + manager, new_empty = self._setUpTest() + + failing_app_names = [ + '', # empty string violates name restriction + 'os', # name of a module in PYTHONPATH causes a name collision + 'askbot', # name of a module in PYTHONPATH causes a name collision + 'bin/bash', # is a file + '/root', # cannot write there + '\/root', + 'usr/local/', + 'usr\/local\/', + 'usr/', + 'usr\/', + 'me\no\like\this', + 'me\\no\\like\\this', + 'me\\\no\\\like\\\this', + ] + for name in failing_app_names: + parameters = {'app_name': name} + e = self.run_complete(manager, parameters) + self.assertIs(type(e), ValueError, parameters) + + valid_app_names = [ + 'validDeployment', + 'askbot_app', + ] + for name in valid_app_names: + parameters = {'app_name': name} + e = self.run_complete(manager, parameters) + self.assertIsNone(e) + + def __test_project_dir_interactive(self): + """The console functions contain endless loops, which impedes + testability. If we mock the console functions, then there is no merrit + in testing interactively at all.""" + manager = FilesystemConfigManager(interactive=True, verbosity=1) + parameters = {'dir_name': ''} + + #with patch('askbot.utils.console.simple_dialog', return_value='validDeployment'), patch('askbot.utils.console.choice_dialog', return_value='yes'): + with patch('builtins.input', new=MockInput('martin', 'yes')): + e = self.run_complete(manager, parameters) + self.assertIsNone(e) From b04a8c352231ac271adfeda587c60bcdbdd644a6 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 30 Jul 2019 23:35:28 +0200 Subject: [PATCH 16/45] Use new validors in dry runs --- askbot/deployment/__init__.py | 19 ++++++++----------- askbot/deployment/parameters/__init__.py | 2 ++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index d4f6333683..035629bfcf 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -228,6 +228,7 @@ def _set_verbosity(self, options): self.verbosity = options.verbosity # I think this logic can be immediately attached to argparse + # it would be a hack though def _set_create_project(self, options): # Currently the --create-project option only changes the installer's # behaviour if one passes "container-uwsgi" as argument @@ -261,8 +262,8 @@ def __call__(self): # this is the main part of the original askbot_setup() options_dict = vars(options) - #from copy import deepcopy - #options_dict_2 = deepcopy(options_dict) + from copy import deepcopy + options_dict_2 = deepcopy(options_dict) if options.force is False: options_dict = collect_missing_options(options_dict) @@ -276,18 +277,14 @@ def __call__(self): # this is the main part of the original askbot_setup() database_engine = database_engine_codes[options.database_engine] options_dict['database_engine'] = database_engine - #from askbot.deployment.parameters import ConfigManagerCollection - - #cm = ConfigManagerCollection(interactive=True, verbosity=2) - #cm.complete(options_dict_2) - - - - if options_dict['dry_run'] == True: + from askbot.deployment.parameters import ConfigManagerCollection + + cm = ConfigManagerCollection(interactive=True, verbosity=2) + cm.complete(options_dict_2) from pprint import pprint pprint(options_dict) - #pprint(options_dict_2) + pprint(options_dict_2) pprint(self.__dict__) raise KeyboardInterrupt diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index 0249e50119..d580665815 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -8,8 +8,10 @@ def __init__(self, interactive=False, verbosity=0): self.interactive = interactive self.database = DbConfigManager(interactive=interactive, verbosity=verbosity) self.cache = CacheConfigManager(interactive=interactive, verbosity=verbosity) + self.filesystem = FilesystemConfigManager(interactive=interactive, verbosity=verbosity) def complete(self, *args, **kwargs): + self.filesystem.complete(*args, **kwargs) self.database.complete(*args, **kwargs) self.cache.complete(*args, **kwargs) From 77a3b98d19275dfa16672f07d5c933eca99dd3c7 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 31 Jul 2019 21:14:11 +0200 Subject: [PATCH 17/45] Fix parser args definition indent and define type=int where appropriate --- askbot/deployment/__init__.py | 129 ++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 62 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 035629bfcf..bc6b8eab89 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -46,10 +46,10 @@ def _add_settings_args(self): """ self.parser.add_argument( - "--logfile-name", - dest="logfile_name", - default='askbot.log', - help="name of the askbot logfile." + "--logfile-name", + dest="logfile_name", + default='askbot.log', + help="name of the askbot logfile." ) self.parser.add_argument( @@ -86,10 +86,10 @@ def _add_setup_args(self): ) self.parser.add_argument( - "--dir-name", "-n", - dest = "dir_name", - default = '', - help = "Directory where you want to install the Django project." + "--dir-name", "-n", + dest = "dir_name", + default = '', + help = "Directory where you want to install the Django project." ) self.parser.add_argument( @@ -100,10 +100,11 @@ def _add_setup_args(self): ) self.parser.add_argument( - "--verbose", "-v", - dest = "verbosity", - default = 1, - help = "verbosity level available values 0, 1, 2." + "--verbose", "-v", + dest = "verbosity", + default = 1, + type=int, + help = "verbosity level available values 0, 1, 2." ) self.parser.add_argument( @@ -143,32 +144,34 @@ def _add_cache_args(self): --cache-password """ self.parser.add_argument('--cache-engine', - dest='cache_engine', - action='store', - default=None, - help='Select which Django cache backend to use. ' - ) + dest='cache_engine', + action='store', + default=None, + type=int, + help='Select which Django cache backend to use.' + ) self.parser.add_argument('--cache-node', - dest='cache_nodes', - action='append', - help='Add cache node to list of nodes. Specify node as :. Can be provided multiple times.' - ) + dest='cache_nodes', + action='append', + help='Add cache node to list of nodes. Specify node as :. Can be provided multiple times.' + ) # only used by redis at the moment self.parser.add_argument('--cache-db', - dest='cache_db', - action='store', - default=1, - help='The name of the cache DB to use.' - ) + dest='cache_db', + action='store', + default=1, + type=int, + help='The name of the cache DB to use.' + ) # only used by redis at the moment self.parser.add_argument('--cache-password', - dest='cache_password', - action='store', - help='The password to connect to the cache.' - ) + dest='cache_password', + action='store', + help='The password to connect to the cache.' + ) def _add_db_args(self): """How to connect to the database @@ -181,48 +184,50 @@ def _add_db_args(self): --db-port """ self.parser.add_argument( - '--db-engine', '-e', - dest='database_engine', - action='store', - choices=DATABASE_ENGINE_CHOICES, - default=None, - help='Database engine, type 1 for PostgreSQL, 2 for SQLite, 3 for MySQL, 4 for Oracle' - ) + '--db-engine', '-e', + dest='database_engine', + action='store', + choices=DATABASE_ENGINE_CHOICES, + default=None, + type=int, + help='Database engine, type 1 for PostgreSQL, 2 for SQLite, 3 for MySQL, 4 for Oracle' + ) self.parser.add_argument( - "--db-name", "-d", - dest = "database_name", - default = None, - help = "The database name Askbot will use" - ) + "--db-name", "-d", + dest = "database_name", + default = None, + help = "The database name Askbot will use" + ) self.parser.add_argument( - "--db-user", "-u", - dest = "database_user", - default = None, - help = "The username Askbot uses to connect to the database" - ) + "--db-user", "-u", + dest = "database_user", + default = None, + help = "The username Askbot uses to connect to the database" + ) self.parser.add_argument( - "--db-password", "-p", - dest = "database_password", - default = None, - help = "The password Askbot uses to connect to the database" - ) + "--db-password", "-p", + dest = "database_password", + default = None, + help = "The password Askbot uses to connect to the database" + ) self.parser.add_argument( - "--db-host", - dest = "database_host", - default = None, - help = "The database host" - ) + "--db-host", + dest = "database_host", + default = None, + help = "The database host" + ) self.parser.add_argument( - "--db-port", - dest = "database_port", - default = None, - help = "The database port" - ) + "--db-port", + dest = "database_port", + default = None, + type=int, + help = "The database port" + ) def _set_verbosity(self, options): self.verbosity = options.verbosity From becede816380d25c85967e5888c5dd54fda17efa Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 31 Jul 2019 23:53:57 +0200 Subject: [PATCH 18/45] AskbotSetup computes parameter-choices form ConfigManagers * added more flexibility w.r.t. verbosity and interactive settings to ConfigManagers * ConfigManagerCollection is now derived from ConfigManager * AskbotSetup loads ConfigManagers during __init__ --- askbot/deployment/__init__.py | 26 ++++++++++------ askbot/deployment/parameters/__init__.py | 39 +++++++++++++++++------- askbot/deployment/parameters/base.py | 29 ++++++++++++++++-- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index bc6b8eab89..35406adf12 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -4,7 +4,6 @@ import os.path import sys -import django from collections import OrderedDict from optparse import OptionParser from argparse import ArgumentParser @@ -14,6 +13,8 @@ from askbot.utils import console from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate +from askbot.deployment.parameters import ConfigManagerCollection + import shutil DATABASE_ENGINE_CHOICES = ('1', '2', '3', '4') @@ -24,10 +25,11 @@ class AskbotSetup: APP_FILES_TO_CREATE = set(path_utils.FILES_TO_CREATE) - set(('manage.py',)) SOURCE_DIR = os.path.dirname(os.path.dirname(__file__)) # a.k.a. ASKBOT_ROOT in settings.py - def __init__(self): + def __init__(self, interactive=True, verbosity=-128): self.parser = ArgumentParser(description="Setup a Django project and app for Askbot") - self.verbosity = -128 + self.verbosity = verbosity self._todo = {} + self.configManagers = ConfigManagerCollection(interactive=interactive, verbosity=verbosity) self._add_arguments() def _add_arguments(self): @@ -104,7 +106,8 @@ def _add_setup_args(self): dest = "verbosity", default = 1, type=int, - help = "verbosity level available values 0, 1, 2." + choices=[0,1,2], + help = "verbosity level with 0 being the lowest" ) self.parser.add_argument( @@ -143,11 +146,14 @@ def _add_cache_args(self): --cache-db --cache-password """ + engines = self.configManagers.configManager('cache').configField('cache_engine').cache_engines + engine_choices = [e[0] for e in engines] self.parser.add_argument('--cache-engine', dest='cache_engine', action='store', default=None, type=int, + choices=engine_choices, help='Select which Django cache backend to use.' ) @@ -183,11 +189,13 @@ def _add_db_args(self): --db-host --db-port """ + engines = self.configManagers.configManager('database').configField('database_engine').database_engines + engine_choices = [e[0] for e in engines] self.parser.add_argument( '--db-engine', '-e', dest='database_engine', action='store', - choices=DATABASE_ENGINE_CHOICES, + choices=engine_choices, default=None, type=int, help='Database engine, type 1 for PostgreSQL, 2 for SQLite, 3 for MySQL, 4 for Oracle' @@ -282,11 +290,9 @@ def __call__(self): # this is the main part of the original askbot_setup() database_engine = database_engine_codes[options.database_engine] options_dict['database_engine'] = database_engine - if options_dict['dry_run'] == True: - from askbot.deployment.parameters import ConfigManagerCollection + if options_dict['dry_run']: - cm = ConfigManagerCollection(interactive=True, verbosity=2) - cm.complete(options_dict_2) + self.configManagers.complete(options_dict_2) from pprint import pprint pprint(options_dict) pprint(options_dict_2) @@ -448,7 +454,7 @@ def deploy_askbot(self, options): ) # set to askbot_setup_orig to return to original installer -askbot_setup = AskbotSetup() +askbot_setup = AskbotSetup(interactive=True, verbosity=1) def askbot_setup_orig(): """basic deployment procedure diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index d580665815..477a274e90 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -1,18 +1,35 @@ -from askbot.deployment.parameters.database import DbConfigManager +from askbot.deployment.parameters.base import ConfigManager from askbot.deployment.parameters.cache import CacheConfigManager +from askbot.deployment.parameters.database import DbConfigManager from askbot.deployment.parameters.filesystem import FilesystemConfigManager -class ConfigManagerCollection(object): +# one could make a case for not deriving ConfigManagerCollection from +# ConfigManager because the collection serves a different purpose than the +# individual manager, but they are still quite similar +class ConfigManagerCollection(ConfigManager): def __init__(self, interactive=False, verbosity=0): - self.verbosity = verbosity - self.interactive = interactive - self.database = DbConfigManager(interactive=interactive, verbosity=verbosity) - self.cache = CacheConfigManager(interactive=interactive, verbosity=verbosity) - self.filesystem = FilesystemConfigManager(interactive=interactive, verbosity=verbosity) + super(ConfigManagerCollection, self).__init__(interactive=interactive, verbosity=verbosity) + self.register('database', DbConfigManager(interactive=interactive, verbosity=verbosity)) + self.register('cache', CacheConfigManager(interactive=interactive, verbosity=verbosity)) + self.register('filesystem', FilesystemConfigManager(interactive=interactive, verbosity=verbosity)) + + def _order(self, keys): + full_set = ['filesystem', 'database', 'cache'] + return [item for item in full_set if item in keys] + + def configManager(self, name): + return super(ConfigManagerCollection, self).configField(name) def complete(self, *args, **kwargs): - self.filesystem.complete(*args, **kwargs) - self.database.complete(*args, **kwargs) - self.cache.complete(*args, **kwargs) + for manager in self._order(self.keys): + handler = self.configManager(manager) + handler.complete(*args, **kwargs) + + # these should never be called. we keep these, just in case + def _remember(self, name, value): + raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') + + def _complete(self, name, value): + raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') -__all__ = [ 'DbConfigManager', 'CacheConfigManager', 'FilesystemConfigManager'] +__all__ = [ 'DbConfigManager', 'CacheConfigManager', 'FilesystemConfigManager', 'ConfigManagerCollection'] diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index 9ebc9e3a9b..d7fef30ab8 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -42,11 +42,36 @@ class ConfigManager(ObjectWithOutput): } def __init__(self, interactive=True, verbosity=1): - super(ConfigManager, self).__init__(verbosity=verbosity) - self.interactive = interactive + self._verbosity = verbosity + self._interactive = interactive self._catalog = dict() self.keys = set() self._managed_config = dict() + super(ConfigManager, self).__init__(verbosity=verbosity) + self.interactive = interactive + + #maybe we also want to do the following for force ... + @property + def interactive(self): + return self._interactive + + @interactive.setter + def interactive(self, interactive): + self._interactive = interactive + for name, handler in self._catalog.items(): + if hasattr(handler,'interactive'): + handler.interactive = interactive + + @property + def verbosity(self): + return self._verbosity + + @verbosity.setter + def verbosity(self, verbosity): + self._verbosity = verbosity + for name, handler in self._catalog.items(): + if hasattr(handler, 'verbosity'): + handler.verbosity = verbosity def register(self, name, handler): """Add the ability to handle a specific install parameter. From 16ede42ec6bc3ea6a5c419a916284a421e348e4b Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 1 Aug 2019 21:57:08 +0200 Subject: [PATCH 19/45] Switch installer to using only ConfigManagers * correctly propagate verbosity settings from the installer instance into the managers and fields * propagate force settings from manager to fields * always run complete() on options (used to be omitted if --force) --- askbot/deployment/__init__.py | 45 ++++++++-------------------- askbot/deployment/parameters/base.py | 15 ++++++++-- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 35406adf12..d9ba579548 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -239,6 +239,7 @@ def _add_db_args(self): def _set_verbosity(self, options): self.verbosity = options.verbosity + self.configManagers.verbosity = options.verbosity # I think this logic can be immediately attached to argparse # it would be a hack though @@ -260,54 +261,35 @@ def __call__(self): # this is the main part of the original askbot_setup() self._set_create_project(options) print_message(messages.DEPLOY_PREAMBLE, self.verbosity) - # the destination directory - directory = path_utils.clean_directory(options.dir_name) - while directory is None: - directory = path_utils.get_install_directory(force=options.force) # i.e. ask the user - options.dir_name = directory - - if options.database_engine not in DATABASE_ENGINE_CHOICES: - options.database_engine = console.choice_dialog( - 'Please select database engine:\n1 - for postgresql, ' - '2 - for sqlite, 3 - for mysql, 4 - oracle', - choices=DATABASE_ENGINE_CHOICES - ) - options_dict = vars(options) - from copy import deepcopy - options_dict_2 = deepcopy(options_dict) + options_dict['dir_name'] = path_utils.clean_directory(options_dict['dir_name']) + options_dict['secret_key'] = '' if options_dict['no_secret_key'] else generate_random_key() - if options.force is False: - options_dict = collect_missing_options(options_dict) + self.configManagers.complete(options_dict) - database_engine_codes = { - '1': 'postgresql_psycopg2', - '2': 'sqlite3', - '3': 'mysql', - '4': 'oracle' - } - database_engine = database_engine_codes[options.database_engine] - options_dict['database_engine'] = database_engine + engines = self.configManagers.configManager( + 'database').configField( + 'database_engine').__class__.database_engines + choice = options_dict['database_engine'] + database_interface = [ e[1] for e in engines if e[0] == choice][0] + options_dict['database_engine'] = database_interface if options_dict['dry_run']: - - self.configManagers.complete(options_dict_2) from pprint import pprint pprint(options_dict) - pprint(options_dict_2) pprint(self.__dict__) raise KeyboardInterrupt self.deploy_askbot(options_dict) - if database_engine == 'postgresql_psycopg2': + if database_interface == 'postgresql_psycopg2': try: import psycopg2 except ImportError: print('\nNEXT STEPS: install python binding for postgresql') print('pip install psycopg2\n') - elif database_engine == 'mysql': + elif database_interface == 'mysql': try: import _mysql except ImportError: @@ -317,7 +299,6 @@ def __call__(self): # this is the main part of the original askbot_setup() except KeyboardInterrupt: print("\n\nAborted.") sys.exit(1) - pass def _install_copy(self, copy_list, forced_overwrite=[], skip_silently=[]): print_message('Copying files:', self.verbosity) @@ -429,7 +410,7 @@ def deploy_askbot(self, options): create_new_project = True if os.path.exists(options['dir_name']) and \ path_utils.has_existing_django_project(options['dir_name']) and \ - options.force is False: + options['force'] is False: create_new_project = False options['staticfiles_app'] = "'django.contrib.staticfiles'," diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index d7fef30ab8..1ca0bd6aae 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -41,16 +41,16 @@ class ConfigManager(ObjectWithOutput): 'eNoValue': 'You must specify a value for "{name}"!', } - def __init__(self, interactive=True, verbosity=1): + def __init__(self, interactive=True, verbosity=1, force=False): self._verbosity = verbosity self._interactive = interactive + self._force = force self._catalog = dict() self.keys = set() self._managed_config = dict() super(ConfigManager, self).__init__(verbosity=verbosity) self.interactive = interactive - #maybe we also want to do the following for force ... @property def interactive(self): return self._interactive @@ -62,6 +62,17 @@ def interactive(self, interactive): if hasattr(handler,'interactive'): handler.interactive = interactive + @property + def force(self): + return self._force + + @force.setter + def force(self, force): + self._force = force + for name, handler in self._catalog.items(): + if hasattr(handler,'force'): + handler.force = force + @property def verbosity(self): return self._verbosity From eafd10ab2e091b08fc8ed5a53c873c869df171ad Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Fri, 2 Aug 2019 21:46:48 +0200 Subject: [PATCH 20/45] Added a few shy test for the Askbot installer * also did some beautifying on the installer itself --- askbot/deployment/__init__.py | 56 ++++++++++++++------------- askbot/tests/test_installer.py | 70 +++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 28 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index d9ba579548..6b6019579e 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -30,6 +30,10 @@ def __init__(self, interactive=True, verbosity=-128): self.verbosity = verbosity self._todo = {} self.configManagers = ConfigManagerCollection(interactive=interactive, verbosity=verbosity) + self.database_engines = self.configManagers.configManager( + 'database').configField( + 'database_engine').database_engines + self._add_arguments() def _add_arguments(self): @@ -189,8 +193,8 @@ def _add_db_args(self): --db-host --db-port """ - engines = self.configManagers.configManager('database').configField('database_engine').database_engines - engine_choices = [e[0] for e in engines] + + engine_choices = [e[0] for e in self.database_engines] self.parser.add_argument( '--db-engine', '-e', dest='database_engine', @@ -237,9 +241,9 @@ def _add_db_args(self): help = "The database port" ) - def _set_verbosity(self, options): - self.verbosity = options.verbosity - self.configManagers.verbosity = options.verbosity + def _set_verbosity(self, v): + self.verbosity = v + self.configManagers.verbosity = v # I think this logic can be immediately attached to argparse # it would be a hack though @@ -247,7 +251,7 @@ def _set_create_project(self, options): # Currently the --create-project option only changes the installer's # behaviour if one passes "container-uwsgi" as argument todo = [ 'django' ] # This is the default as Askbot has always worked - wish = str.lower(options.create_project) + wish = str.lower(options['create_project']) if wish in [ 'no', 'none', 'false', '0']: todo = [ 'nothing' ] elif wish == 'container-uwsgi': @@ -257,47 +261,45 @@ def _set_create_project(self, options): def __call__(self): # this is the main part of the original askbot_setup() try: options = self.parser.parse_args() - self._set_verbosity(options) + options = vars(options) + self._set_verbosity(options['verbosity']) self._set_create_project(options) print_message(messages.DEPLOY_PREAMBLE, self.verbosity) - options_dict = vars(options) - options_dict['dir_name'] = path_utils.clean_directory(options_dict['dir_name']) - options_dict['secret_key'] = '' if options_dict['no_secret_key'] else generate_random_key() - self.configManagers.complete(options_dict) + options['dir_name'] = path_utils.clean_directory(options['dir_name']) + options['secret_key'] = '' if options['no_secret_key'] else generate_random_key() + + self.configManagers.complete(options) - engines = self.configManagers.configManager( - 'database').configField( - 'database_engine').__class__.database_engines - choice = options_dict['database_engine'] - database_interface = [ e[1] for e in engines if e[0] == choice][0] - options_dict['database_engine'] = database_interface + database_interface = [ e[1] for e in self.database_engines + if e[0] == options['database_engine'] ][0] + options['database_engine'] = database_interface - if options_dict['dry_run']: - from pprint import pprint - pprint(options_dict) - pprint(self.__dict__) + if options['dry_run']: + from pprint import pformat + print_message(pformat(options), self.verbosity) + print_message(pformat(self.__dict__), self.verbosity) raise KeyboardInterrupt - self.deploy_askbot(options_dict) + self.deploy_askbot(options) if database_interface == 'postgresql_psycopg2': try: import psycopg2 except ImportError: - print('\nNEXT STEPS: install python binding for postgresql') - print('pip install psycopg2\n') + print_message('\nNEXT STEPS: install python binding for postgresql', self.verbosity) + print_message('pip install psycopg2\n', self.verbosity) elif database_interface == 'mysql': try: import _mysql except ImportError: - print('\nNEXT STEP: install python binding for mysql') - print('pip install mysql-python\n') + print_message('\nNEXT STEP: install python binding for mysql', self.verbosity) + print_message('pip install mysql-python\n', self.verbosity) except KeyboardInterrupt: - print("\n\nAborted.") + print_message("\n\nAborted.", self.verbosity) sys.exit(1) def _install_copy(self, copy_list, forced_overwrite=[], skip_silently=[]): diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index f892b62085..00726d6c80 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -2,7 +2,7 @@ from askbot.deployment import AskbotSetup from askbot.deployment.parameters import * -from unittest.mock import patch +from unittest.mock import patch, MagicMock class MockInput: def __init__(self, *args): @@ -419,3 +419,71 @@ def __test_project_dir_interactive(self): with patch('builtins.input', new=MockInput('martin', 'yes')): e = self.run_complete(manager, parameters) self.assertIsNone(e) + +class MainInstallerTests(AskbotTestCase): + def setUp(self): + self.installer = AskbotSetup(interactive=True, verbosity=0) + + def test_propagate_attributes(self): + collection = self.installer.configManagers + managers = [collection.configManager(key) for key in collection.keys] + fields = [m.configField(k) for m in managers for k in m.keys] + + has_verbosity = [ self.installer, collection ] + has_verbosity.extend(managers) + has_verbosity.extend(fields) + for obj in has_verbosity: + self.assertEqual(obj.verbosity, self.installer.verbosity) + + for new_verbosity in [2, 5, 13, 23, 42, 666, 1337, 16061, self.installer.verbosity]: + self.installer._set_verbosity(new_verbosity) + for obj in has_verbosity: + self.assertEqual(obj.verbosity, new_verbosity) + + interactive_fields = [m.configField(k) + for m in managers + for k in m.keys + if hasattr(m.configField(k),'interactive')] + has_interactive = [ collection ] + has_interactive.extend(managers) + has_interactive.extend(interactive_fields) + for obj in has_interactive: + self.assertEqual(obj.interactive, collection.interactive) + + for new_interactive in [True, False, True, collection.interactive]: + collection.interactive = new_interactive + for obj in has_interactive: + self.assertEqual(obj.interactive, new_interactive) + + def test_flow_dry_run(self): + self.installer.configManagers.interactive=False + # minimal_viable_argument_set + mva = ['--dir-name', '/tmp/AskbotTestDir', + '--db-engine', '2', + '--db-name', '/tmp/AskbotTest.db', + '--cache-engine', '3'] + + run_opts = ['--dry-run', '-v', '0'] + with patch('sys.exit') as mock: + opts = self.installer.parser.parse_args(mva + run_opts) + parse_args = MagicMock(name='parse_args', return_value=opts) + self.installer.parser.parse_args = parse_args + self.installer() + self.assertEqual(mock.call_count, 1) + + def test_flow_skip_deploy(self): + mva = ['--dir-name', '/tmp/AskbotTestDir', + '--db-engine', '2', + '--db-name', '/tmp/AskbotTest.db', + '--cache-engine', '3'] + run_opts = ['-v', '0'] + + opts = self.installer.parser.parse_args(mva + run_opts) + parse_args = MagicMock(name='parse_args', return_value=opts) + deploy_askbot = MagicMock() + self.installer.parser.parse_args = parse_args + self.installer.deploy_askbot = deploy_askbot + try: + self.installer() + except Exception as e: + self.fail(f'Running the installer raised {e}') From a46e4ed971de3955d3e7eb4a9b721b64990e787f Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Fri, 2 Aug 2019 23:05:06 +0200 Subject: [PATCH 21/45] Added deployment unit test for the Askbot installer --- askbot/tests/test_installer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 00726d6c80..1f7ce82a4c 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -2,7 +2,7 @@ from askbot.deployment import AskbotSetup from askbot.deployment.parameters import * -from unittest.mock import patch, MagicMock +from unittest.mock import patch, MagicMock, mock_open class MockInput: def __init__(self, *args): @@ -487,3 +487,16 @@ def test_flow_skip_deploy(self): self.installer() except Exception as e: self.fail(f'Running the installer raised {e}') + + def test_flow_mock_deployment(self): + mva = ['--dir-name', '/tmp/AskbotTestDir', + '--db-engine', '2', + '--db-name', '/tmp/AskbotTest.db', + '--cache-engine', '3'] + run_opts = ['-v', '0'] + opts = self.installer.parser.parse_args(mva + run_opts) + parse_args = MagicMock(name='parse_args', return_value=opts) + self.installer.parser.parse_args = parse_args + fake_open = mock_open(read_data='foobar') + with patch('askbot.deployment.path_utils.create_path'), patch('askbot.deployment.path_utils.touch'), patch('shutil.copy'), patch('builtins.open', fake_open): + self.installer() From 32f3796834ed55f942bf7e404a88a4a26a3e5b65 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Fri, 2 Aug 2019 23:24:10 +0200 Subject: [PATCH 22/45] Use the app-name parameter, make --force defunct and housekeeping --- askbot/deployment/__init__.py | 267 ++-------------------------------- 1 file changed, 12 insertions(+), 255 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 6b6019579e..126374873e 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -4,25 +4,21 @@ import os.path import sys -from collections import OrderedDict -from optparse import OptionParser + from argparse import ArgumentParser from askbot.deployment import messages from askbot.deployment.messages import print_message from askbot.deployment import path_utils -from askbot.utils import console from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate from askbot.deployment.parameters import ConfigManagerCollection import shutil -DATABASE_ENGINE_CHOICES = ('1', '2', '3', '4') - class AskbotSetup: - PROJECT_FILES_TO_CREATE = set(('manage.py',)) - APP_FILES_TO_CREATE = set(path_utils.FILES_TO_CREATE) - set(('manage.py',)) + PROJECT_FILES_TO_CREATE = {'manage.py'} + APP_FILES_TO_CREATE = set(path_utils.FILES_TO_CREATE) - {'manage.py'} SOURCE_DIR = os.path.dirname(os.path.dirname(__file__)) # a.k.a. ASKBOT_ROOT in settings.py def __init__(self, interactive=True, verbosity=-128): @@ -118,7 +114,7 @@ def _add_setup_args(self): "--force", dest="force", action='store_true', - help="Force overwrite settings.py file" + help="(DEFUNCT!) Force overwrite settings.py file" ) self.parser.add_argument( @@ -262,6 +258,7 @@ def __call__(self): # this is the main part of the original askbot_setup() try: options = self.parser.parse_args() options = vars(options) + options['force'] = False # disable the --force switch! self._set_verbosity(options['verbosity']) self._set_create_project(options) print_message(messages.DEPLOY_PREAMBLE, self.verbosity) @@ -353,10 +350,10 @@ def _create_new_django_project(self, install_dir, options): path_utils.touch(log_file) self._install_copy(copy_me, skip_silently=path_utils.BLANK_FILES) - def _create_new_django_app(self, app_name, options): - options['askbot_site'] = options['dir_name'] - options['askbot_app'] = app_name - app_dir = os.path.join(options['dir_name'], app_name) + def _create_new_django_app(self, options): + options['askbot_site'] = options['dir_name'] # b/c the jinja template uses askbot_site + options['askbot_app'] = options['app_name'] # b/c the jinja template uses askbot_app + app_dir = os.path.join(options['dir_name'], options['app_name']) create_me = [ app_dir ] copy_me = list() @@ -415,13 +412,13 @@ def deploy_askbot(self, options): options['force'] is False: create_new_project = False - options['staticfiles_app'] = "'django.contrib.staticfiles'," - options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' + options['staticfiles_app'] = "'django.contrib.staticfiles'," # Fixme: move this into the template + options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' # Fixme: move this into the template if create_new_project is True: self._create_new_django_project(options['dir_name'], options) - self._create_new_django_app('askbot_app', options) + self._create_new_django_app(options) help_file = path_utils.get_path_to_help_file() @@ -438,243 +435,3 @@ def deploy_askbot(self, options): # set to askbot_setup_orig to return to original installer askbot_setup = AskbotSetup(interactive=True, verbosity=1) - -def askbot_setup_orig(): - """basic deployment procedure - asks user several questions, then either creates - new deployment (in the case of new installation) - or gives hints on how to add askbot to an existing - Django project - """ - parser = OptionParser(usage = "%prog [options]") - - parser.add_option( - "-v", "--verbose", - dest = "verbosity", - type = "int", - default = 1, - help = "verbosity level available values 0, 1, 2." - ) - - parser.add_option( - "-n", "--dir-name", - dest = "dir_name", - default = None, - help = "Directory where you want to install." - ) - - parser.add_option( - '-e', '--db-engine', - dest='database_engine', - action='store', - type='choice', - choices=DATABASE_ENGINE_CHOICES, - default=None, - help='Database engine, type 1 for postgresql, 2 for sqlite, 3 for mysql' - ) - - parser.add_option( - "-d", "--db-name", - dest = "database_name", - default = None, - help = "The database name" - ) - - parser.add_option( - "-u", "--db-user", - dest = "database_user", - default = None, - help = "The database user" - ) - - parser.add_option( - "-p", "--db-password", - dest = "database_password", - default = None, - help = "the database password" - ) - - parser.add_option( - "--db-host", - dest = "database_host", - default = None, - help = "the database host" - ) - - parser.add_option( - "--db-port", - dest = "database_port", - default = None, - help = "the database host" - ) - - parser.add_option( - "--logfile-name", - dest="logfile_name", - default='askbot.log', - help="name of the askbot logfile." - ) - - parser.add_option( - "--append-settings", - dest = "local_settings", - default = '', - help = "Extra settings file to append custom settings" - ) - - parser.add_option( - "--force", - dest="force", - action='store_true', - default=False, - help = "Force overwrite settings.py file" - ) - parser.add_option( - "--no-secret-key", - dest="no_secret_key", - action='store_true', - default=False, - help="Don't generate a secret key. (not recommended)" - ) - - try: - options = parser.parse_args()[0] - - #ask users to give missing parameters - #todo: make this more explicit here - if options.verbosity >= 1: - print(messages.DEPLOY_PREAMBLE) - - directory = path_utils.clean_directory(options.dir_name) - while directory is None: - directory = path_utils.get_install_directory(force=options.force) - options.dir_name = directory - - if options.database_engine not in DATABASE_ENGINE_CHOICES: - options.database_engine = console.choice_dialog( - 'Please select database engine:\n1 - for postgresql, ' - '2 - for sqlite, 3 - for mysql, 4 - oracle', - choices=DATABASE_ENGINE_CHOICES - ) - - options_dict = vars(options) - if options.force is False: - options_dict = collect_missing_options(options_dict) - - database_engine_codes = { - '1': 'postgresql_psycopg2', - '2': 'sqlite3', - '3': 'mysql', - '4': 'oracle' - } - database_engine = database_engine_codes[options.database_engine] - options_dict['database_engine'] = database_engine - - deploy_askbot(options_dict) - - if database_engine == 'postgresql_psycopg2': - try: - import psycopg2 - except ImportError: - print('\nNEXT STEPS: install python binding for postgresql') - print('pip install psycopg2\n') - elif database_engine == 'mysql': - try: - import _mysql - except ImportError: - print('\nNEXT STEP: install python binding for mysql') - print('pip install mysql-python\n') - - except KeyboardInterrupt: - print("\n\nAborted.") - sys.exit(1) - - -#separated all the directory creation process to make it more useful -def deploy_askbot(options): - """function that creates django project files, - all the neccessary directories for askbot, - and the log file - """ - create_new_project = True - if os.path.exists(options['dir_name']): - if path_utils.has_existing_django_project(options['dir_name']): - create_new_project = bool(options['force']) - - path_utils.create_path(options['dir_name']) - - options['staticfiles_app'] = "'django.contrib.staticfiles'," - - options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' - - verbosity = options['verbosity'] - - path_utils.deploy_into( - options['dir_name'], - new_project=create_new_project, - verbosity=verbosity, - context=options - ) - - help_file = path_utils.get_path_to_help_file() - - if create_new_project: - print_message( - messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}, - verbosity - ) - else: - print_message( - messages.HOW_TO_ADD_ASKBOT_TO_DJANGO % {'help_file': help_file}, - verbosity - ) - -def collect_missing_options(options_dict): - options_dict['secret_key'] = '' if options_dict['no_secret_key'] else generate_random_key() - if options_dict['database_engine'] == '2':#sqlite - if options_dict['database_name']: - return options_dict - while True: - value = console.simple_dialog( - 'Please enter database file name' - ) - database_file_name = None - if os.path.isfile(value): - message = 'file %s exists, use it anyway?' % value - if console.get_yes_or_no(message) == 'yes': - database_file_name = value - elif os.path.isdir(value): - print('%s is a directory, choose another name' % value) - elif value in path_utils.FILES_TO_CREATE: - print('name %s cannot be used for the database name' % value) - elif value == path_utils.LOG_DIR_NAME: - print('name %s cannot be used for the database name' % value) - else: - database_file_name = value - - if database_file_name: - options_dict['database_name'] = database_file_name - return options_dict - - else:#others - db_keys = OrderedDict([ - ('database_name', True), - ('database_user', True), - ('database_password', True), - ('database_host', False), - ('database_port', False) - ]) - for key, required in list(db_keys.items()): - if options_dict[key] is None: - key_name = key.replace('_', ' ') - fmt_string = '\nPlease enter %s' - if not required: - fmt_string += ' (press "Enter" to use the default value)' - - value = console.simple_dialog( - fmt_string % key_name, - required=db_keys[key] - ) - - options_dict[key] = value - return options_dict From 3f01102cecbc9d37872ee43ea1adb73c302d8237 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Mon, 5 Aug 2019 22:41:27 +0200 Subject: [PATCH 23/45] Extended tox ini to have PGSQL and SQLITE based deployment tests --- tox.ini | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index b858ad46df..47f10a4ada 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = py{3.6,3.7,pypy} lint - setup + setup{pg,sl} [testenv] deps = @@ -23,24 +23,46 @@ deps = prospector commands = prospector askbot -0 -[testenv:setup] +[testenv:setuppg] basepython = python3.7 deps = -raskbot_requirements.txt + requests_oauthlib psycopg2-binary dj-database-url whitelist_externals = mkdir rm psql - +setenv = DJANGO_SETTINGS_MODULE = askbot_app.settings commands_pre = mkdir -p {toxinidir}/deploy_askbot psql -h localhost -p 5432 -U postgres -c "DROP DATABASE IF EXISTS deploy_askbot" psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE deploy_askbot OWNER='askbot'" commands = - askbot-setup --dir-name={toxinidir}/deploy_askbot --db-engine=1 --db-name=deploy_askbot --db-host=localhost --db-port=5432 --db-user=askbot --db-password='askB0T!' + askbot-setup --dir-name={toxinidir}/deploy_askbot --db-engine=1 --db-name=deploy_askbot --db-host=localhost --db-port=5432 --db-user=askbot --db-password='askB0T!' --cache-engine=3 + python {toxinidir}/deploy_askbot/manage.py migrate --noinput + python {toxinidir}/deploy_askbot/manage.py collectstatic --noinput +commands_post = + rm -rf {toxinidir}/deploy_askbot + rm -rf {toxinidir}/static + +[testenv:setupsl] +basepython = + python3.7 +deps = + -raskbot_requirements.txt + requests_oauthlib + dj-database-url +whitelist_externals = + mkdir + rm +setenv = DJANGO_SETTINGS_MODULE = askbot_app.settings +commands_pre = + mkdir -p {toxinidir}/deploy_askbot +commands = + askbot-setup --dir-name={toxinidir}/deploy_askbot --db-engine=2 --db-name={toxinidir}/deploy_askbot/askbot.db --cache-engine=3 python {toxinidir}/deploy_askbot/manage.py migrate --noinput python {toxinidir}/deploy_askbot/manage.py collectstatic --noinput commands_post = From 528ec7525fcc51880641b6fd0023fe8a92dcac96 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sun, 11 Aug 2019 00:40:08 +0200 Subject: [PATCH 24/45] Commited WiP files for publication * both files will be removed before this work is finished * these are my current thoughts and first implementation * this is not integrated into Askbot. You can read deployments.md locally and run dev_django.py in a virtualenv to play around with the classes. --- askbot/deployment/deployments.md | 157 +++++++++++++++ askbot/deployment/dev_django.py | 330 +++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 askbot/deployment/deployments.md create mode 100644 askbot/deployment/dev_django.py diff --git a/askbot/deployment/deployments.md b/askbot/deployment/deployments.md new file mode 100644 index 0000000000..7454ef8bcd --- /dev/null +++ b/askbot/deployment/deployments.md @@ -0,0 +1,157 @@ +# On Askbot deployments +This is a model of what askbot-setup usually should deploy + + { + 'manage.py': None + 'log': { + 'askbot.log': None + }, + 'askbot_site': { + 'settings.py': None, + '__init__.py': None, + 'celery_app.py': None, + 'urls.py': None, + }, + 'askbot_app': { + 'cron': { + 'send_email_alerts.sh': None + }, + 'doc': #copy of doc/ from askbot module + 'upfiles': {} + } + } + +None values mean key is a file created by askbot setup, values of type +dict meand key is a directory. The syntax error means 'doc' is complicated. + +`askbot-setup -n /path/to/destdir` +means "install Askbot into this directory". -n therefore provides the root +directory for the Django project. Everything else must be placed in this +directory. +=> $(addprefix /path/to/destdir, askbot_site, log, askbot_app, manage.py) + +Let's look at each of the four top level objects +* askbot_site, +* log, +* askbot_app and +* manage.py + +## askbot_site + +Django's usual approach is to duplicate the root dir's basename. Deploying +into `/path/to/destdir` should yield `/path/to/destdir/destdir` for 'askbot-site'. + +If `manage.py` already exists, then we are deploying into an existing Django +project. Usually this means `/path/to/destdir/destdir` does already exist. +Overwriting any of the existing files would break the existing project and +potentially leaves us with a mix of our freshly deployed Askbot files and some +custom files that were there before the Askbot deployment but weren't +overwritten during the deployment, because they don't exist in vanilla Askbot. +Why would anybody want to deploy Askbot into an existing project? + +##### Reason 1: People want to add Askbot to their existing project. +In this case + overwriting is not what we want. Adding Askbot to an existing project means + adding 'askbot_app' and adding Askbot to urls.py. One also has to (probably) add Celery, + add Askbot to INSTALLED_APPs and ensure that settings.py has everything else + required by Askbot. There is no reason to believe simply overwriting anything + in 'askbot_site' gets us any closer to this goal than not changing 'askbot_site' + at all. +##### Reason 2: The admin wants to reset an existing Askbot project to its original state. +In that case + we shouldn't just overwrite 'askbot_site' but delete the entire directory + and rebuilt it from scratch, to ensure there aren't any remnants of the old + deployment. +##### Reason 3: The admin wants to migrate from an old Askbot installation/version to a new one. + In that case we + most likely want to keep a lot of the existing config around and not(!) + overwrite it. +##### Reason 4: The admin wants to put Askbot exactly into this directory, because other directories are not an option. +In this case, but at the same time not for reasons 1, 2 or 3, seems like a + corner case. Would it be too much to ask the admin to clean out the + directory before deploying Askbot there? I think not. +##### Conclusion: Overwriting 'askbot_site' is never really useful. +#### Approach +With reasons 1 and 3 we deploy 'askbot_site' into a differently named directory. +As a result there will two valid site configs in the project root. The one that +has been there before and the one created by `askbot-setup`. Which config is +effectively used, is determined through `manage.py` or at least through the +environment variable `DJANGO_SETTINGS_MODULE`. With this setup, admins can +**manually** merge files or **manually** change `manage.py` to point to the +newly deployed `settings.py`. + +With reasons 2 and 4 we want to create a clean 'askbot_site' directory. +Effectively, we want to replace the existing site setting. For the best result +we rename the original site config directory so that we can install Askbot's +'askbot_site' directory as if we were doing a green field deployment. This +avoids unwanted side effects. With this approach, admins can refer to their old +deployments or even modify `manage.py` to switch back to the old deployment. + +## log + +Django default projects do not have a dedicated log dir. This is something +Askbot specific. Consequently, log should be moved into 'askbot_app', especially +when people want to add Askbot to their existing project. Should we ever +overwrite existing log files? NO! We can rename them or even `logrotate` them. +But deleting seems wreckless and not useful.The log dir name and its location are currently +hard coded into Askbot. This is weired because it seems there is no requirement +for it. Everything else can be modified, why not the log dir name and location? + +#####Approach: +Make log dir configurable and rotate existing files. Notify the user + when rotating files. + +## askbot_app + +This is really Askbot deployment specific and only Askbot deployment specific. +A running Askbot refers to files or directories at this location. Stuff in here +only interacts with other apps if they want to interact with this Askbot deployment. + +##### cron/ +This directory holds the one shell script that we want cron to run. We should +also add our suggestion of a corresponding crontab, for users to add. Why would +anybody want to overwrite it? +- Customization +- Upgrade +- Reset + +These seem valid and realtively frequent. `askbot-setup` should have commands for +running `crontab` and have the capability to conditionallly overwrite existing configs. + +##### doc/ +This is a copy of the documentation's source code. A copy is probably not all +that helpful, but putting some form of consumable documentation into the +deployment may be helpful. It doesn't seem reasonable that users would modify +Askbot's documentation source on a per deployment basis. Rendered versions, +however can/should change with Askbot versions and the form the users want to +consume (PDF, HTML, whatever). Enabling askbot-setup to render the documentation +into `doc/` would be nice to have. As currently we are only copying source, +it is probably more sensible to link to the source code and with changing +versions `askbot-setup` should be able to update the link. + +##### upfiles/ +Initially empty, populated by Askbot users, not admins/operators, but the +consumers of the deployed Askbot. Files in this directory are references by +database entries and therefore cannot be moved/deleted without compromising +Askbot's data integrity. However, management operations like "backup" and "restore" +may be nice. Do those exist? This is nothing the installer does though. + +The +installer creates this directory and if it already exists ... then it exists. +Maybe a warning message that there is data in this directory which may be +unrelated to the database entries would be nice. There is also the potential for +a reset/fresh install which warrants a wipe of this directory. It can be +sensible to have the wipe in the installer. + +##### Conclusion: Having the installer work with existing files is useful in many cases. +##### Approach +Make the installer work under the assumption that it does a clean + field deployment. Give it a switch to work with existing files and directories. + Issue warnings or raise exceptions if attempting to overwrite files with + the overwrite switch set. + +## manage.py + + There is nothing fancy about this file. On fresh installations we want to + write it and make it point to 'askbot_site.settings'. On any other use case??? + diff --git a/askbot/deployment/dev_django.py b/askbot/deployment/dev_django.py new file mode 100644 index 0000000000..fbcfe51a26 --- /dev/null +++ b/askbot/deployment/dev_django.py @@ -0,0 +1,330 @@ +""" +The main purpose of this code is to do a one-shot deployment of Askbot. It is +built on the premise that the chosen deployment is viable and possible. If an +error occurs, it is not this code's task to remedy the issue. If an error +occurs the deployment is considered as failed. Ideally, all the work this code +did up to the error is undone. Yet, this code has no means to undo anything. +""" + +from askbot.deployment.messages import print_message +from askbot.deployment.template_loader import DeploymentTemplate +import os.path +import shutil + +class AskbotDeploymentError(Exception): + """Use this when something goes wrong while deploying Askbot""" + +class DeployObject(object): + def __init__(self, name, src_path=None, dst_path=None): + self._verbosity = 2 + self.name = name + self._src_path = src_path + self._dst_path = dst_path + + @property + def verbosity(self): + return self._verbosity + + @verbosity.setter + def verbosity(self, value): + self._verbosity = value + + @property + def src_path(self): + return self._src_path + + @src_path.setter + def src_path(self, value): + self._src_path = value + + @property + def dst_path(self): + return self._dst_path + + @dst_path.setter + def dst_path(self, value): + self._dst_path = value + + @property + def src(self): + return os.path.join(self.src_path, self.name) + + @property + def dst(self): + return os.path.join(self.dst_path, self.name) + + def _deploy_now(self): + """virtual protected""" + + def deploy(self): + print_message(f'* {self.dst} from {self.src}', self.verbosity) + try: + self._deploy_now() + except AskbotDeploymentError as e: + print_message(e, self.verbosity) + + +class DeployFile(DeployObject): + """This class collects all logic w.r.t. writing files. It has to be + subclassed and the subclasses must/should overwrite _deploy_now to call + one of the methods defined in this class. At the time of this writing, this + is either _render_with_jinja2 or _copy.""" + def __init__(self, name, src_path=None, dst_path=None): + super(DeployFile, self).__init__(name, src_path, dst_path) + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + # the different approaches for force and skip feel a little odd. + def __validate(self): + matches = [self.dst for c in self.forced_overwrite + if self.dst.endswith(f'{os.path.sep}{c}')] + exists = os.path.exists(self.dst) + force = len(matches) > 0 + skip = self.dst.split(os.path.sep)[-1] in self.skip_silently + return exists, force, skip + + def _render_with_jinja2(self, context): + exists, force, skip = self.__validate() + if exists: + raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') + template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended + template.tmpl_path = self.src + with open(self.dst, 'w+') as output_file: + output_file.write(template.render(context)) + + def _copy(self): + exists, force, skip = self.__validate() + if exists: + if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. + return + elif force: + print_message(' ^^^ forced overwrite!', self.verbosity) + else: + raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') + shutil.copy(self.src, self.dst) + + +class DeployDir(DeployObject): + def __init__(self, name, parent=None, *content): + super(DeployDir, self).__init__(name) + self.content = list(content) + if parent is not None: + self.dst_path = self.__clean_directory(parent) + + @property + def src_path(self): + return super().src_path + + @src_path.setter + def src_path(self, value): + value = self.__clean_directory(value) + self._src_path = value + for child in self.content: + child.src_path = value + + @property + def dst_path(self): + return super().dst_path + + @dst_path.setter + def dst_path(self, value): + value = self.__clean_directory(value) + self._dst_path = value + for child in self.content: + child.dst_path = self.dst + + @property + def context(self): + raise AttributeError(f'{self.__class__} does not store context information') + + @property + def skip_silently(self): + raise AttributeError(f'{self.__class__} does not store file skip information') + + @property + def forced_overwrite(self): + raise AttributeError(f'{self.__class__} does not store file overwrite information') + + # maybe deepcopy() in the following three setters. maybe there's an advantage in not deepcopy()ing here. time will tell. + @context.setter + def context(self, value): + for child in self.content: # in theory, we only want this for directories and rendered files. + child.context = value + + @skip_silently.setter + def skip_silently(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.skip_silently = value + + @forced_overwrite.setter + def forced_overwrite(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.forced_overwrite = value + + ##################### + ## from path_utils ## + ##################### + def __create_path(self): + """equivalent to mkdir -p""" + if os.path.isdir(self.dst): + return + elif os.path.exists(self.dst): + raise AskbotDeploymentError('expect directory or a non-existing path') + else: + os.makedirs(self.dst) + + @staticmethod + def __clean_directory(directory): + """Returns normalized absolute path to the directory + regardless of whether it exists or not + or ``None`` - if the path is a file or if ``directory`` + parameter is ``None``""" + if directory is None: + return None + + directory = os.path.normpath(directory) + directory = os.path.abspath(directory) + + #if os.path.isfile(directory): + # print(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) + # return None + return directory + + def _deploy_now(self): + self.__create_path() + + def deploy(self): + self._deploy_now() # we don't try-except this, b/c input validation is supposed to identify problems before this code runs. If an exception is raised here, it's an issue that must be handled elsewhere. + for node in self.content: + node.deploy() + +class RenderedFile(DeployFile): + def _deploy_now(self): + self._render_with_jinja2(self.context) + + @property + def src(self): + return f'{super().src}.jinja2' + +class CopiedFile(DeployFile): + def _deploy_now(self): + self._copy() + +class Directory(DeployDir): + pass + +class LinkedDir(DeployDir): + pass + + +class DeployableComponent(object): + """These constitue sensible deployment chunks of Askbot. For instance, + one would never deploy just a settings.py, because Django projects need + additional files as well.""" + default_name = 'unset' + contents = dict() + + def __init__(self, name=None, contents=None): + name = name if name is not None else self.__class__.default_name + contents = contents if contents is not None else self.__class__.contents + + self.name = name + self.contents = contents + self._src_dir = None + self._dst_dir = None + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + def _grow_deployment_tree(self, component): + todo = list() + for name, deployable in component.items(): + if isinstance(deployable,dict): + branch = self._grow_deployment_tree(deployable) + todo.append( + Directory( + name, + None, + *branch + )) + else: + todo.append(deployable(name)) + return todo + + def _root_deployment_tree(self, tree): + root = Directory(self.name, None, *tree) + root.src_path = self.src_dir + root.dst_path = self.dst_dir + root.context = self.context + root.forced_overwrite = self.forced_overwrite + root.skip_silently = self.skip_silently + return root + + @property + def src_dir(self): + return self._src_dir + + @src_dir.setter + def src_dir(self, value): + self._src_dir = value + + @property + def dst_dir(self): + return self._dst_dir + + @dst_dir.setter + def dst_dir(self, value): + self._dst_dir = value + + def deploy(self): + """Recursively deploy all DeployItems we know about. This method is + meant to be overwritten by subclasses to provide more fine grained + configurations to individual branches and/or nodes.""" + tree = self._grow_deployment_tree(self.contents) + root = self._root_deployment_tree(tree) + root.deploy() + + +class AskbotSite(DeployableComponent): + default_name = 'askbot_site' + contents = { + 'settings.py': RenderedFile, + '__init__.py': CopiedFile, + 'celery_app.py': CopiedFile, + 'urls.py': CopiedFile, + } + +class AskbotApp(DeployableComponent): + default_name = 'askbot_app' + contents = { + 'cron': { + 'send_email_alerts.sh': CopiedFile, + }, + 'doc': LinkedDir, + 'upfiles': {}, + } + +class ProjectRoot(DeployableComponent): + contents = { 'manage.py': RenderedFile } + + def __init__(self, install_path): + dirname, basename = os.path.split(install_path) + if len(basename) == 0: + dirname, basename = os.path.split(dirname) + super(ProjectRoot, self).__init__(basename) + self.dst_dir = dirname + +if __name__ == '__main__': + test = CopiedFile('testfile', '/tmp/foo', '/tmp/bar') + test.deploy() + test = RenderedFile('testfile01', '/tmp/foo', '/tmp/bar') + test.context = {'x': 'World'} + test.deploy() + test = Directory('baz', '/tmp') + test.deploy() + askbot_site = AskbotSite() + askbot_app = AskbotApp() + project_root = ProjectRoot('/tmp/project_root_install_dir') + project_root.src_dir = '/tmp/foo' + project_root.deploy() From fd6e7d96d59fe3a6af52ffb06e35151dd57ac353 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Tue, 13 Aug 2019 17:23:54 +0200 Subject: [PATCH 25/45] created directory structure for deployables --- askbot/deployment/deployables/__init__.py | 16 + askbot/deployment/deployables/components.py | 98 ++++++ askbot/deployment/deployables/objects.py | 211 +++++++++++++ askbot/deployment/dev_django.py | 317 +------------------- 4 files changed, 326 insertions(+), 316 deletions(-) create mode 100644 askbot/deployment/deployables/__init__.py create mode 100644 askbot/deployment/deployables/components.py create mode 100644 askbot/deployment/deployables/objects.py diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py new file mode 100644 index 0000000000..8e9b6b63be --- /dev/null +++ b/askbot/deployment/deployables/__init__.py @@ -0,0 +1,16 @@ +""" +The main purpose of this code is to do a one-shot deployment of Askbot. It is +built on the premise that the chosen deployment is viable and possible. If an +error occurs, it is not this code's task to remedy the issue. If an error +occurs the deployment is considered as failed. Ideally, all the work this code +did up to the error is undone. Yet, this code has no means to undo anything. +""" + +from .objects import RenderedFile, CopiedFile, Directory, LinkedDir +from .components import AskbotApp, AskbotSite, ProjectRoot + +class AskbotDeploymentError(Exception): + """Use this when something goes wrong while deploying Askbot""" + +__all__ = ['RenderedFile', 'CopiedFile', 'Directory', 'LinkedDir', + 'AskbotApp', 'AskbotSite', 'ProjectRoot', 'AskbotDeploymentError'] diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py new file mode 100644 index 0000000000..19070ad638 --- /dev/null +++ b/askbot/deployment/deployables/components.py @@ -0,0 +1,98 @@ + +from askbot.deployment.deployables.objects import * + +class DeployableComponent(object): + """These constitute sensible deployment chunks of Askbot. For instance, + one would never deploy just a settings.py, because Django projects need + additional files as well.""" + default_name = 'unset' + contents = dict() + + def __init__(self, name=None, contents=None): + name = name if name is not None else self.__class__.default_name + contents = contents if contents is not None else self.__class__.contents + + self.name = name + self.contents = contents + self._src_dir = None + self._dst_dir = None + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + def _grow_deployment_tree(self, component): + todo = list() + for name, deployable in component.items(): + if isinstance(deployable,dict): + branch = self._grow_deployment_tree(deployable) + todo.append( + Directory( + name, + None, + *branch + )) + else: + todo.append(deployable(name)) + return todo + + def _root_deployment_tree(self, tree): + root = Directory(self.name, None, *tree) + root.src_path = self.src_dir + root.dst_path = self.dst_dir + root.context = self.context + root.forced_overwrite = self.forced_overwrite + root.skip_silently = self.skip_silently + return root + + @property + def src_dir(self): + return self._src_dir + + @src_dir.setter + def src_dir(self, value): + self._src_dir = value + + @property + def dst_dir(self): + return self._dst_dir + + @dst_dir.setter + def dst_dir(self, value): + self._dst_dir = value + + def deploy(self): + """Recursively deploy all DeployItems we know about. This method is + meant to be overwritten by subclasses to provide more fine grained + configurations to individual branches and/or nodes.""" + tree = self._grow_deployment_tree(self.contents) + root = self._root_deployment_tree(tree) + root.deploy() + +class AskbotSite(DeployableComponent): + default_name = 'askbot_site' + contents = { + 'settings.py': RenderedFile, + '__init__.py': CopiedFile, + 'celery_app.py': CopiedFile, + 'urls.py': CopiedFile, + } + +class AskbotApp(DeployableComponent): + default_name = 'askbot_app' + contents = { + 'cron': { + 'send_email_alerts.sh': CopiedFile, + }, + 'doc': LinkedDir, + 'upfiles': {}, + } + +class ProjectRoot(DeployableComponent): + contents = { 'manage.py': RenderedFile } + + def __init__(self, install_path): + dirname, basename = os.path.split(install_path) + if len(basename) == 0: + dirname, basename = os.path.split(dirname) + super(ProjectRoot, self).__init__(basename) + self.dst_dir = dirname diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py new file mode 100644 index 0000000000..b98915409f --- /dev/null +++ b/askbot/deployment/deployables/objects.py @@ -0,0 +1,211 @@ +import os.path +from askbot.deployment.messages import print_message +from askbot.deployment.deployables import AskbotDeploymentError + + +class DeployObject(object): + """Base class for filesystem objects, i.e. files and directories, that can + be/are deployed using askbot-setup.""" + def __init__(self, name, src_path=None, dst_path=None): + self._verbosity = 2 + self.name = name + self._src_path = src_path + self._dst_path = dst_path + + @property + def verbosity(self): + return self._verbosity + + @verbosity.setter + def verbosity(self, value): + self._verbosity = value + + @property + def src_path(self): + return self._src_path + + @src_path.setter + def src_path(self, value): + self._src_path = value + + @property + def dst_path(self): + return self._dst_path + + @dst_path.setter + def dst_path(self, value): + self._dst_path = value + + @property + def src(self): + return os.path.join(self.src_path, self.name) + + @property + def dst(self): + return os.path.join(self.dst_path, self.name) + + def _deploy_now(self): + """virtual protected""" + + def deploy(self): + """The main method of this class. DeployableComponents call this method + to have this object do the filesystem operations which deploy a single + file or directory.""" + print_message(f'* {self.dst} from {self.src}', self.verbosity) + try: + self._deploy_now() + except AskbotDeploymentError as e: + print_message(e, self.verbosity) + +class DeployFile(DeployObject): + """This class collects all logic w.r.t. writing files. It has to be + subclassed and the subclasses must/should overwrite _deploy_now to call + one of the methods defined in this class. At the time of this writing, this + is either _render_with_jinja2 or _copy.""" + def __init__(self, name, src_path=None, dst_path=None): + super(DeployFile, self).__init__(name, src_path, dst_path) + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + # the different approaches for force and skip feel a little odd. + def __validate(self): + matches = [self.dst for c in self.forced_overwrite + if self.dst.endswith(f'{os.path.sep}{c}')] + exists = os.path.exists(self.dst) + force = len(matches) > 0 + skip = self.dst.split(os.path.sep)[-1] in self.skip_silently + return exists, force, skip + + def _render_with_jinja2(self, context): + exists, force, skip = self.__validate() + if exists: + raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') + template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended + template.tmpl_path = self.src + with open(self.dst, 'w+') as output_file: + output_file.write(template.render(context)) + + def _copy(self): + exists, force, skip = self.__validate() + if exists: + if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. + return + elif force: + print_message(' ^^^ forced overwrite!', self.verbosity) + else: + raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') + shutil.copy(self.src, self.dst) + + +class DeployDir(DeployObject): + def __init__(self, name, parent=None, *content): + super(DeployDir, self).__init__(name) + self.content = list(content) + if parent is not None: + self.dst_path = self.__clean_directory(parent) + + @property + def src_path(self): + return super().src_path + + @src_path.setter + def src_path(self, value): + value = self.__clean_directory(value) + self._src_path = value + for child in self.content: + child.src_path = value + + @property + def dst_path(self): + return super().dst_path + + @dst_path.setter + def dst_path(self, value): + value = self.__clean_directory(value) + self._dst_path = value + for child in self.content: + child.dst_path = self.dst + + @property + def context(self): + raise AttributeError(f'{self.__class__} does not store context information') + + @property + def skip_silently(self): + raise AttributeError(f'{self.__class__} does not store file skip information') + + @property + def forced_overwrite(self): + raise AttributeError(f'{self.__class__} does not store file overwrite information') + + # maybe deepcopy() in the following three setters. maybe there's an advantage in not deepcopy()ing here. time will tell. + @context.setter + def context(self, value): + for child in self.content: # in theory, we only want this for directories and rendered files. + child.context = value + + @skip_silently.setter + def skip_silently(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.skip_silently = value + + @forced_overwrite.setter + def forced_overwrite(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.forced_overwrite = value + + ##################### + ## from path_utils ## + ##################### + def __create_path(self): + """equivalent to mkdir -p""" + if os.path.isdir(self.dst): + return + elif os.path.exists(self.dst): + raise AskbotDeploymentError('expect directory or a non-existing path') + else: + os.makedirs(self.dst) + + @staticmethod + def __clean_directory(directory): + """Returns normalized absolute path to the directory + regardless of whether it exists or not + or ``None`` - if the path is a file or if ``directory`` + parameter is ``None``""" + if directory is None: + return None + + directory = os.path.normpath(directory) + directory = os.path.abspath(directory) + + #if os.path.isfile(directory): + # print(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) + # return None + return directory + + def _deploy_now(self): + self.__create_path() + + def deploy(self): + self._deploy_now() # we don't try-except this, b/c input validation is supposed to identify problems before this code runs. If an exception is raised here, it's an issue that must be handled elsewhere. + for node in self.content: + node.deploy() + +class RenderedFile(DeployFile): + def _deploy_now(self): + self._render_with_jinja2(self.context) + + @property + def src(self): + return f'{super().src}.jinja2' + +class CopiedFile(DeployFile): + def _deploy_now(self): + self._copy() + +class Directory(DeployDir): + pass + +class LinkedDir(DeployDir): + pass diff --git a/askbot/deployment/dev_django.py b/askbot/deployment/dev_django.py index fbcfe51a26..df7d1a5126 100644 --- a/askbot/deployment/dev_django.py +++ b/askbot/deployment/dev_django.py @@ -1,319 +1,4 @@ -""" -The main purpose of this code is to do a one-shot deployment of Askbot. It is -built on the premise that the chosen deployment is viable and possible. If an -error occurs, it is not this code's task to remedy the issue. If an error -occurs the deployment is considered as failed. Ideally, all the work this code -did up to the error is undone. Yet, this code has no means to undo anything. -""" - -from askbot.deployment.messages import print_message -from askbot.deployment.template_loader import DeploymentTemplate -import os.path -import shutil - -class AskbotDeploymentError(Exception): - """Use this when something goes wrong while deploying Askbot""" - -class DeployObject(object): - def __init__(self, name, src_path=None, dst_path=None): - self._verbosity = 2 - self.name = name - self._src_path = src_path - self._dst_path = dst_path - - @property - def verbosity(self): - return self._verbosity - - @verbosity.setter - def verbosity(self, value): - self._verbosity = value - - @property - def src_path(self): - return self._src_path - - @src_path.setter - def src_path(self, value): - self._src_path = value - - @property - def dst_path(self): - return self._dst_path - - @dst_path.setter - def dst_path(self, value): - self._dst_path = value - - @property - def src(self): - return os.path.join(self.src_path, self.name) - - @property - def dst(self): - return os.path.join(self.dst_path, self.name) - - def _deploy_now(self): - """virtual protected""" - - def deploy(self): - print_message(f'* {self.dst} from {self.src}', self.verbosity) - try: - self._deploy_now() - except AskbotDeploymentError as e: - print_message(e, self.verbosity) - - -class DeployFile(DeployObject): - """This class collects all logic w.r.t. writing files. It has to be - subclassed and the subclasses must/should overwrite _deploy_now to call - one of the methods defined in this class. At the time of this writing, this - is either _render_with_jinja2 or _copy.""" - def __init__(self, name, src_path=None, dst_path=None): - super(DeployFile, self).__init__(name, src_path, dst_path) - self.context = dict() - self.skip_silently = list() - self.forced_overwrite = list() - - # the different approaches for force and skip feel a little odd. - def __validate(self): - matches = [self.dst for c in self.forced_overwrite - if self.dst.endswith(f'{os.path.sep}{c}')] - exists = os.path.exists(self.dst) - force = len(matches) > 0 - skip = self.dst.split(os.path.sep)[-1] in self.skip_silently - return exists, force, skip - - def _render_with_jinja2(self, context): - exists, force, skip = self.__validate() - if exists: - raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') - template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended - template.tmpl_path = self.src - with open(self.dst, 'w+') as output_file: - output_file.write(template.render(context)) - - def _copy(self): - exists, force, skip = self.__validate() - if exists: - if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. - return - elif force: - print_message(' ^^^ forced overwrite!', self.verbosity) - else: - raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') - shutil.copy(self.src, self.dst) - - -class DeployDir(DeployObject): - def __init__(self, name, parent=None, *content): - super(DeployDir, self).__init__(name) - self.content = list(content) - if parent is not None: - self.dst_path = self.__clean_directory(parent) - - @property - def src_path(self): - return super().src_path - - @src_path.setter - def src_path(self, value): - value = self.__clean_directory(value) - self._src_path = value - for child in self.content: - child.src_path = value - - @property - def dst_path(self): - return super().dst_path - - @dst_path.setter - def dst_path(self, value): - value = self.__clean_directory(value) - self._dst_path = value - for child in self.content: - child.dst_path = self.dst - - @property - def context(self): - raise AttributeError(f'{self.__class__} does not store context information') - - @property - def skip_silently(self): - raise AttributeError(f'{self.__class__} does not store file skip information') - - @property - def forced_overwrite(self): - raise AttributeError(f'{self.__class__} does not store file overwrite information') - - # maybe deepcopy() in the following three setters. maybe there's an advantage in not deepcopy()ing here. time will tell. - @context.setter - def context(self, value): - for child in self.content: # in theory, we only want this for directories and rendered files. - child.context = value - - @skip_silently.setter - def skip_silently(self, value): - for child in self.content: # in practice, we only need this for directories and copied files. - child.skip_silently = value - - @forced_overwrite.setter - def forced_overwrite(self, value): - for child in self.content: # in practice, we only need this for directories and copied files. - child.forced_overwrite = value - - ##################### - ## from path_utils ## - ##################### - def __create_path(self): - """equivalent to mkdir -p""" - if os.path.isdir(self.dst): - return - elif os.path.exists(self.dst): - raise AskbotDeploymentError('expect directory or a non-existing path') - else: - os.makedirs(self.dst) - - @staticmethod - def __clean_directory(directory): - """Returns normalized absolute path to the directory - regardless of whether it exists or not - or ``None`` - if the path is a file or if ``directory`` - parameter is ``None``""" - if directory is None: - return None - - directory = os.path.normpath(directory) - directory = os.path.abspath(directory) - - #if os.path.isfile(directory): - # print(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) - # return None - return directory - - def _deploy_now(self): - self.__create_path() - - def deploy(self): - self._deploy_now() # we don't try-except this, b/c input validation is supposed to identify problems before this code runs. If an exception is raised here, it's an issue that must be handled elsewhere. - for node in self.content: - node.deploy() - -class RenderedFile(DeployFile): - def _deploy_now(self): - self._render_with_jinja2(self.context) - - @property - def src(self): - return f'{super().src}.jinja2' - -class CopiedFile(DeployFile): - def _deploy_now(self): - self._copy() - -class Directory(DeployDir): - pass - -class LinkedDir(DeployDir): - pass - - -class DeployableComponent(object): - """These constitue sensible deployment chunks of Askbot. For instance, - one would never deploy just a settings.py, because Django projects need - additional files as well.""" - default_name = 'unset' - contents = dict() - - def __init__(self, name=None, contents=None): - name = name if name is not None else self.__class__.default_name - contents = contents if contents is not None else self.__class__.contents - - self.name = name - self.contents = contents - self._src_dir = None - self._dst_dir = None - self.context = dict() - self.skip_silently = list() - self.forced_overwrite = list() - - def _grow_deployment_tree(self, component): - todo = list() - for name, deployable in component.items(): - if isinstance(deployable,dict): - branch = self._grow_deployment_tree(deployable) - todo.append( - Directory( - name, - None, - *branch - )) - else: - todo.append(deployable(name)) - return todo - - def _root_deployment_tree(self, tree): - root = Directory(self.name, None, *tree) - root.src_path = self.src_dir - root.dst_path = self.dst_dir - root.context = self.context - root.forced_overwrite = self.forced_overwrite - root.skip_silently = self.skip_silently - return root - - @property - def src_dir(self): - return self._src_dir - - @src_dir.setter - def src_dir(self, value): - self._src_dir = value - - @property - def dst_dir(self): - return self._dst_dir - - @dst_dir.setter - def dst_dir(self, value): - self._dst_dir = value - - def deploy(self): - """Recursively deploy all DeployItems we know about. This method is - meant to be overwritten by subclasses to provide more fine grained - configurations to individual branches and/or nodes.""" - tree = self._grow_deployment_tree(self.contents) - root = self._root_deployment_tree(tree) - root.deploy() - - -class AskbotSite(DeployableComponent): - default_name = 'askbot_site' - contents = { - 'settings.py': RenderedFile, - '__init__.py': CopiedFile, - 'celery_app.py': CopiedFile, - 'urls.py': CopiedFile, - } - -class AskbotApp(DeployableComponent): - default_name = 'askbot_app' - contents = { - 'cron': { - 'send_email_alerts.sh': CopiedFile, - }, - 'doc': LinkedDir, - 'upfiles': {}, - } - -class ProjectRoot(DeployableComponent): - contents = { 'manage.py': RenderedFile } - - def __init__(self, install_path): - dirname, basename = os.path.split(install_path) - if len(basename) == 0: - dirname, basename = os.path.split(dirname) - super(ProjectRoot, self).__init__(basename) - self.dst_dir = dirname +from askbot.deployment.deployables import * if __name__ == '__main__': test = CopiedFile('testfile', '/tmp/foo', '/tmp/bar') From 099909457e1a1238bc0e98f761a573c409dc60f8 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 14 Aug 2019 20:35:44 +0200 Subject: [PATCH 26/45] Added unit tests for deployables.objects and made them work --- askbot/deployment/deployables/__init__.py | 4 +- askbot/deployment/deployables/objects.py | 9 +- askbot/tests/test_installer.py | 121 ++++++++++++++++++++++ 3 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 askbot/tests/test_installer.py diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py index 8e9b6b63be..9dbfd45b31 100644 --- a/askbot/deployment/deployables/__init__.py +++ b/askbot/deployment/deployables/__init__.py @@ -6,11 +6,9 @@ did up to the error is undone. Yet, this code has no means to undo anything. """ -from .objects import RenderedFile, CopiedFile, Directory, LinkedDir +from .objects import RenderedFile, CopiedFile, Directory, LinkedDir, AskbotDeploymentError from .components import AskbotApp, AskbotSite, ProjectRoot -class AskbotDeploymentError(Exception): - """Use this when something goes wrong while deploying Askbot""" __all__ = ['RenderedFile', 'CopiedFile', 'Directory', 'LinkedDir', 'AskbotApp', 'AskbotSite', 'ProjectRoot', 'AskbotDeploymentError'] diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index b98915409f..e51972f5ca 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -1,6 +1,10 @@ import os.path +import shutil from askbot.deployment.messages import print_message -from askbot.deployment.deployables import AskbotDeploymentError +from askbot.deployment.template_loader import DeploymentTemplate + +class AskbotDeploymentError(Exception): + """Use this when something goes wrong while deploying Askbot""" class DeployObject(object): @@ -83,8 +87,9 @@ def _render_with_jinja2(self, context): raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended template.tmpl_path = self.src + content = template.render(context) # do not put this in the context. If this statement fails, we do not want an empty self.dst to be created! with open(self.dst, 'w+') as output_file: - output_file.write(template.render(context)) + output_file.write(content) def _copy(self): exists, force, skip = self.__validate() diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py new file mode 100644 index 0000000000..85235a71bd --- /dev/null +++ b/askbot/tests/test_installer.py @@ -0,0 +1,121 @@ +from askbot.tests.utils import AskbotTestCase +from askbot.deployment import AskbotSetup +#from askbot.deployment.parameters import * +from askbot.deployment.deployables import * +import tempfile +import os.path + +from unittest.mock import patch, MagicMock, mock_open + + + +class DeployObjectsTest(AskbotTestCase): + def setUp(self): + """creates two temporary directories: + - project_root, not contents + - setup_templates, source files for installation tests: + - jinja2_template a minimal jinja2 template that can be rendered + - text_file a plain text file + setUp() also sets self.jinja2_target, which is the filename (not the + path!) that must be used with RenderedFile, so that it will load + jinja2_template. + """ + self.project_root = tempfile.TemporaryDirectory() + self.setup_templates = tempfile.TemporaryDirectory() + + self.jinja2_template = tempfile.NamedTemporaryFile(suffix='.jinja2', delete=False, dir=self.setup_templates.name) + jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" + self.jinja2_template.write(jinja2.encode('utf-8')) + self.jinja2_template.close() + + self.jinja2_target = os.path.splitext( + os.path.basename( + self.jinja2_template.name))[0] + + self.text_file = tempfile.NamedTemporaryFile(delete=False, dir=self.setup_templates.name) + self.text_file.write("Hello World.\n".encode('utf-8')) + self.text_file.close() + + def tearDown(self): + del(self.project_root) + del(self.setup_templates) + + def test_individualCopiedFile(self): + basename = os.path.basename(self.text_file.name) + + test = CopiedFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + self.assertRaises(FileNotFoundError, test.deploy) + + test = CopiedFile(basename, self.setup_templates.name + '_source_does_not_exist', self.project_root.name) + self.assertRaises(FileNotFoundError, test.deploy) + + test = CopiedFile(basename + '_bogus', self.setup_templates.name, self.project_root.name) + self.assertRaises(FileNotFoundError, test.deploy) + + # this should just work and copy the file + test = CopiedFile(basename, self.setup_templates.name, self.project_root.name) + test.deploy() + + new_file = os.path.join(self.project_root.name, basename) + try: + with open(new_file, 'r') as file: + buf = file.read() + self.assertGreater(len(buf), 0) + except FileNotFoundError as e: + self.fail(FileNotFoundError('Copying the file did not work!')) + + # do it again. Should yield a user notification and then succeed by + # not doing anything. + test.deploy() + + def test_individualRenderedFile(self): + basename = self.jinja2_target + + test = RenderedFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + self.assertRaises(FileNotFoundError, test.deploy) + + test = RenderedFile(basename, self.setup_templates.name + '_source_does_not_exist', self.project_root.name) + self.assertRaises(FileNotFoundError, test.deploy) + + test = RenderedFile(basename + '_bogus', self.setup_templates.name, self.project_root.name) + self.assertRaises(FileNotFoundError, test.deploy) + + test = RenderedFile(basename, self.setup_templates.name, self.project_root.name) + test.context = {'world': 'World'} + test.deploy() + + new_file = os.path.join(self.project_root.name, basename) + try: + with open(new_file, 'r') as file: + buf = file.read() + self.assertGreater(len(buf), 0) + except FileNotFoundError as e: + self.fail(FileNotFoundError('Rendering the file did not work!')) + + # do it again. Should yield a user notification and then succeed by + # not doing anything. + test.deploy() + + + def test_individualDirectory(self): + basename = 'foobar' + + test = Directory(basename, self.project_root.name) + test.deploy() + + new_dir = os.path.join(self.project_root.name, basename) + + self.assertTrue(os.path.exists(new_dir)) + self.assertTrue(os.path.isdir(new_dir)) + + # do it again. This should succeed by + # not doing anything. + test.deploy() + + + + +if __name__ == '__main__': + askbot_site = AskbotSite() + askbot_app = AskbotApp() + From a93715eda86f2f46cb3671859f84aff66a11ee4b Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 14 Aug 2019 22:04:42 +0200 Subject: [PATCH 27/45] Added unittests for DeployableComponents --- askbot/tests/test_installer.py | 68 ++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 85235a71bd..5207fe6809 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -113,9 +113,71 @@ def test_individualDirectory(self): test.deploy() +class DeployableComponentsTest(AskbotTestCase): + def setUp(self): + """creates two temporary directories: + - project_root, not contents + - setup_templates, source files for installation tests: + All supported DeployableComponents are registered here and then searched + for the files they want to deploy. corresponding templates are then + generated in setup_templates, so that the deployment may succeed. + """ + + jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" + hello = "Hello World.\n" + + self.project_root = tempfile.TemporaryDirectory() + self.setup_templates = tempfile.TemporaryDirectory() + x, y, z = AskbotApp(), AskbotSite(), ProjectRoot(self.project_root.name) + self.deployableComponents = { + x.name: x, + y.name: y, + z.name: z, + } + + def flatten_components(components): + found = [i for i in components + if i[1] is RenderedFile + or i[1] is CopiedFile] + + for descent in [list(i[1].items()) for i in components + if isinstance(i[1], dict) ]: + found.extend( + flatten_components(descent) + ) + return found + + for cname, comp in self.deployableComponents.items(): + for fname, ftype in flatten_components(list(comp.contents.items())): + if ftype is RenderedFile: + with open(os.path.join( + self.setup_templates.name, + f'{fname}.jinja2'), 'wb') as f: + f.write(jinja2.encode('utf-8')) + elif ftype is CopiedFile: + with open(os.path.join( + self.setup_templates.name, fname), 'wb') as f: + f.write(hello.encode('utf-8')) -if __name__ == '__main__': - askbot_site = AskbotSite() - askbot_app = AskbotApp() + def tearDown(self): + del(self.project_root) + del(self.setup_templates) + + def test_ProjectRoot(self): + test = ProjectRoot(self.project_root.name) + test.src_dir = self.setup_templates.name + test.deploy() + + def test_AskbotSite(self): + test = AskbotSite() + test.src_dir = self.setup_templates.name + test.dst_dir = self.project_root.name + test.deploy() + + def test_AskbotApp(self): + test = AskbotApp() + test.src_dir = self.setup_templates.name + test.dst_dir = self.project_root.name + test.deploy() From a37d07ee4a807c853821d5cc3536c6c4c05678df Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 14 Aug 2019 22:30:45 +0200 Subject: [PATCH 28/45] Refined deployables unittests --- askbot/tests/test_installer.py | 72 +++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 5207fe6809..f6988c23b8 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -20,12 +20,14 @@ def setUp(self): path!) that must be used with RenderedFile, so that it will load jinja2_template. """ + self.jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" + self.hello = "Hello World.\n" + self.project_root = tempfile.TemporaryDirectory() self.setup_templates = tempfile.TemporaryDirectory() self.jinja2_template = tempfile.NamedTemporaryFile(suffix='.jinja2', delete=False, dir=self.setup_templates.name) - jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" - self.jinja2_template.write(jinja2.encode('utf-8')) + self.jinja2_template.write(self.jinja2.encode('utf-8')) self.jinja2_template.close() self.jinja2_target = os.path.splitext( @@ -33,12 +35,12 @@ def setUp(self): self.jinja2_template.name))[0] self.text_file = tempfile.NamedTemporaryFile(delete=False, dir=self.setup_templates.name) - self.text_file.write("Hello World.\n".encode('utf-8')) + self.text_file.write(self.hello.encode('utf-8')) self.text_file.close() def tearDown(self): - del(self.project_root) - del(self.setup_templates) + del self.project_root + del self.setup_templates def test_individualCopiedFile(self): basename = os.path.basename(self.text_file.name) @@ -58,10 +60,11 @@ def test_individualCopiedFile(self): new_file = os.path.join(self.project_root.name, basename) try: - with open(new_file, 'r') as file: + with open(new_file, 'rb') as file: buf = file.read() self.assertGreater(len(buf), 0) - except FileNotFoundError as e: + self.assertEqual(buf.decode('utf-8'), self.hello) + except FileNotFoundError: self.fail(FileNotFoundError('Copying the file did not work!')) # do it again. Should yield a user notification and then succeed by @@ -86,10 +89,11 @@ def test_individualRenderedFile(self): new_file = os.path.join(self.project_root.name, basename) try: - with open(new_file, 'r') as file: + with open(new_file, 'rb') as file: buf = file.read() self.assertGreater(len(buf), 0) - except FileNotFoundError as e: + self.assertNotEqual(buf.decode('utf-8'), self.jinja2) + except FileNotFoundError: self.fail(FileNotFoundError('Rendering the file did not work!')) # do it again. Should yield a user notification and then succeed by @@ -114,6 +118,18 @@ def test_individualDirectory(self): class DeployableComponentsTest(AskbotTestCase): + def _flatten_components(self, components): + found = [i for i in components + if i[1] is RenderedFile + or i[1] is CopiedFile] + + for descent in [list(i[1].items()) for i in components + if isinstance(i[1], dict)]: + found.extend( + self._flatten_components(descent) + ) + return found + def setUp(self): """creates two temporary directories: - project_root, not contents @@ -123,8 +139,8 @@ def setUp(self): generated in setup_templates, so that the deployment may succeed. """ - jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" - hello = "Hello World.\n" + self.jinja2 = "Hello {{ world }}! <-- This should read Hello World!\n" + self.hello = "Hello World.\n" self.project_root = tempfile.TemporaryDirectory() self.setup_templates = tempfile.TemporaryDirectory() @@ -136,48 +152,48 @@ def setUp(self): z.name: z, } - def flatten_components(components): - found = [i for i in components - if i[1] is RenderedFile - or i[1] is CopiedFile] - - for descent in [list(i[1].items()) for i in components - if isinstance(i[1], dict) ]: - found.extend( - flatten_components(descent) - ) - return found - for cname, comp in self.deployableComponents.items(): - for fname, ftype in flatten_components(list(comp.contents.items())): + for fname, ftype in self._flatten_components(list(comp.contents.items())): if ftype is RenderedFile: with open(os.path.join( self.setup_templates.name, f'{fname}.jinja2'), 'wb') as f: - f.write(jinja2.encode('utf-8')) + f.write(self.jinja2.encode('utf-8')) elif ftype is CopiedFile: with open(os.path.join( self.setup_templates.name, fname), 'wb') as f: - f.write(hello.encode('utf-8')) + f.write(self.hello.encode('utf-8')) def tearDown(self): - del(self.project_root) - del(self.setup_templates) + del self.project_root + del self.setup_templates def test_ProjectRoot(self): test = ProjectRoot(self.project_root.name) test.src_dir = self.setup_templates.name test.deploy() + comp = self.deployableComponents[test.name] + for name, value in comp.contents.items(): + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) + def test_AskbotSite(self): test = AskbotSite() test.src_dir = self.setup_templates.name test.dst_dir = self.project_root.name test.deploy() + comp = self.deployableComponents[test.name] + for name, value in comp.contents.items(): + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, comp.name, name))) + def test_AskbotApp(self): test = AskbotApp() test.src_dir = self.setup_templates.name test.dst_dir = self.project_root.name test.deploy() + comp = self.deployableComponents[test.name] + for name, value in comp.contents.items(): + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, comp.name, name))) + From 726dc7c7aa3fe7d001e2ab9e317fad4f6b963c51 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 14 Aug 2019 23:41:40 +0200 Subject: [PATCH 29/45] Integrated deployables into installer (BROKEN!) ! Apparently the installer (and tox) requre the env var SECRET_KEY to be set in current 0.11.x upstream * created deploy_askbot_new() and use it as a drop in replacement for deploy_askbot() * modified deployed directory structure to include Evgeny's suggestion from https://github.com/ASKBOT/askbot-devel/pull/833 * Added new deployable EmptyFile to accomodate the empty log file --- askbot/deployment/__init__.py | 44 +++++++++++++++++- askbot/deployment/deployables/__init__.py | 4 +- askbot/deployment/deployables/components.py | 8 ++-- askbot/deployment/deployables/objects.py | 16 +++++++ askbot/setup_templates/manage.py.jinja2 | 10 ++++ askbot/setup_templates/send_email_alerts.sh | 1 + askbot/tests/test_installer.py | 51 +++++++++++++++++++++ 7 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 askbot/setup_templates/manage.py.jinja2 create mode 120000 askbot/setup_templates/send_email_alerts.sh diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 9772633ecc..4f77810a29 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -14,6 +14,7 @@ from askbot.utils import console from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate +import askbot.deployment.deployables as deployable import shutil DATABASE_ENGINE_CHOICES = ('1', '2', '3', '4') @@ -236,7 +237,7 @@ def __call__(self): # this is the main part of the original askbot_setup() database_engine = database_engine_codes[options.database_engine] options_dict['database_engine'] = database_engine - self.deploy_askbot(options_dict) + self.deploy_askbot_new(options_dict) if database_engine == 'postgresql_psycopg2': try: @@ -390,6 +391,47 @@ def deploy_askbot(self, options): self.verbosity ) + def deploy_askbot_new(self, options): + create_new_project = True + if os.path.exists(options['dir_name']) and \ + path_utils.has_existing_django_project(options['dir_name']) and \ + options.force is False: + create_new_project = False + + options['staticfiles_app'] = "'django.contrib.staticfiles'," + options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' + + project = deployable.ProjectRoot(options['dir_name']) + site = deployable.AskbotSite(project.name) + + if create_new_project is True: + project.src_dir = os.path.join(self.SOURCE_DIR, 'setup_templates') + project.contents.update({ + path_utils.LOG_DIR_NAME: {options['logfile_name']: deployable.EmptyFile} + }) + project.context = {'settings_path': f'{site.name}.settings'} + project.deploy() + + site.src_dir = os.path.join(self.SOURCE_DIR, 'setup_templates') + site.dst_dir = options['dir_name'] + site.context.update(options) + site.deploy() + + help_file = path_utils.get_path_to_help_file() + + if create_new_project: + print_message( + messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}, + self.verbosity + ) + else: + print_message( + messages.HOW_TO_ADD_ASKBOT_TO_DJANGO % {'help_file': help_file}, + self.verbosity + ) + + + # set to askbot_setup_orig to return to original installer askbot_setup = AskbotSetup() diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py index 9dbfd45b31..afa06252a7 100644 --- a/askbot/deployment/deployables/__init__.py +++ b/askbot/deployment/deployables/__init__.py @@ -6,9 +6,9 @@ did up to the error is undone. Yet, this code has no means to undo anything. """ -from .objects import RenderedFile, CopiedFile, Directory, LinkedDir, AskbotDeploymentError +from .objects import RenderedFile, CopiedFile, EmptyFile, Directory, LinkedDir, AskbotDeploymentError from .components import AskbotApp, AskbotSite, ProjectRoot -__all__ = ['RenderedFile', 'CopiedFile', 'Directory', 'LinkedDir', +__all__ = ['RenderedFile', 'CopiedFile', 'EmptyFile', 'Directory', 'LinkedDir', 'AskbotApp', 'AskbotSite', 'ProjectRoot', 'AskbotDeploymentError'] diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py index 19070ad638..6921f3994c 100644 --- a/askbot/deployment/deployables/components.py +++ b/askbot/deployment/deployables/components.py @@ -75,11 +75,16 @@ class AskbotSite(DeployableComponent): '__init__.py': CopiedFile, 'celery_app.py': CopiedFile, 'urls.py': CopiedFile, + 'django.wsgi': CopiedFile, } class AskbotApp(DeployableComponent): default_name = 'askbot_app' + contents = {} + +class ProjectRoot(DeployableComponent): contents = { + 'manage.py': RenderedFile, 'cron': { 'send_email_alerts.sh': CopiedFile, }, @@ -87,9 +92,6 @@ class AskbotApp(DeployableComponent): 'upfiles': {}, } -class ProjectRoot(DeployableComponent): - contents = { 'manage.py': RenderedFile } - def __init__(self, install_path): dirname, basename = os.path.split(install_path) if len(basename) == 0: diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index e51972f5ca..69d90a3fe4 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -102,6 +102,18 @@ def _copy(self): raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') shutil.copy(self.src, self.dst) + ##################### + ## from path_utils ## + ##################### + def _touch(self, times=None): + """implementation of unix ``touch`` in python""" + #http://stackoverflow.com/questions/1158076/implement-touch-using-python + fhandle = open(self.dst, 'a') + try: + os.utime(self.dst, times) + finally: + fhandle.close() + class DeployDir(DeployObject): def __init__(self, name, parent=None, *content): @@ -209,6 +221,10 @@ class CopiedFile(DeployFile): def _deploy_now(self): self._copy() +class EmptyFile(DeployFile): + def _deploy_now(self): + self._touch() + class Directory(DeployDir): pass diff --git a/askbot/setup_templates/manage.py.jinja2 b/askbot/setup_templates/manage.py.jinja2 new file mode 100644 index 0000000000..c2b55412db --- /dev/null +++ b/askbot/setup_templates/manage.py.jinja2 @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ settings_path }}") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/askbot/setup_templates/send_email_alerts.sh b/askbot/setup_templates/send_email_alerts.sh new file mode 120000 index 0000000000..0677dcf8da --- /dev/null +++ b/askbot/setup_templates/send_email_alerts.sh @@ -0,0 +1 @@ +../cron/askbot_cron_job \ No newline at end of file diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index f6988c23b8..78e0e1c17c 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -100,6 +100,26 @@ def test_individualRenderedFile(self): # not doing anything. test.deploy() + def test_individualEmptyFile(self): + basename = os.path.basename(self.text_file.name) + + test = EmptyFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + self.assertRaises(FileNotFoundError, test.deploy) + + # this should just work and copy the file + test = EmptyFile(basename, self.setup_templates.name, self.project_root.name) + test.deploy() + + new_file = os.path.join(self.project_root.name, basename) + try: + with open(new_file, 'rb') as file: + buf = file.read() + except FileNotFoundError: + self.fail(FileNotFoundError('Copying the file did not work!')) + + # do it again. Should yield a user notification and then succeed by + # not doing anything. + test.deploy() def test_individualDirectory(self): basename = 'foobar' @@ -197,3 +217,34 @@ def test_AskbotApp(self): for name, value in comp.contents.items(): self.assertTrue(os.path.exists(os.path.join(self.project_root.name, comp.name, name))) + def test_addFileBeforeDeploy(self): + test = ProjectRoot(self.project_root.name) + + another_file = os.path.join(self.setup_templates.name, 'additional.file') + with open(another_file, 'wb') as f: + f.write(self.hello.encode('utf-8')) + test.contents.update({'additional.file': CopiedFile}) + + test.src_dir = self.setup_templates.name + test.deploy() + + comp = self.deployableComponents[test.name] + for name, value in comp.contents.items(): + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, 'additional.file'))) + + def test_addDirBeforeDeploy(self): + test = ProjectRoot(self.project_root.name) + + another_file = os.path.join(self.setup_templates.name, 'additional.file') + with open(another_file, 'wb') as f: + f.write(self.hello.encode('utf-8')) + test.contents.update({'additionalDir': {'additional.file': CopiedFile}}) + + test.src_dir = self.setup_templates.name + test.deploy() + + comp = self.deployableComponents[test.name] + for name, value in comp.contents.items(): + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) + self.assertTrue(os.path.exists(os.path.join(self.project_root.name, 'additionalDir', 'additional.file'))) From 46d2829f6f09128dfe539f83ea4a94f6de61988d Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Wed, 14 Aug 2019 23:48:27 +0200 Subject: [PATCH 30/45] Modified settings.py.jinja2 to reflect the most recent change in directory layout * see https://github.com/ASKBOT/askbot-devel/pull/833 --- askbot/setup_templates/settings.py.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askbot/setup_templates/settings.py.jinja2 b/askbot/setup_templates/settings.py.jinja2 index 2efa5cc0ce..77b3fd81c4 100644 --- a/askbot/setup_templates/settings.py.jinja2 +++ b/askbot/setup_templates/settings.py.jinja2 @@ -84,7 +84,7 @@ ASKBOT_LANGUAGE_MODE = 'single-lang' #'single-lang', 'url-lang', 'user-lang' # Absolute path to the directory that holds uploaded media # Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(os.path.dirname(__file__), 'askbot', 'upfiles') +MEDIA_ROOT = os.path.join(os.path.dirname(__file__), 'upfiles') MEDIA_URL = '/upfiles/' # url to uploaded media. This is expected to start with a / STATIC_URL = '/m/'#this must be different from MEDIA_URL USE_LOCAL_FONTS = False From 47afce7410ac6ad40a53944420c5217e3e86be81 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 15 Aug 2019 10:42:54 +0200 Subject: [PATCH 31/45] Adopted ObjectWithOutput from structure_installer_using_classes --- askbot/deployment/deployables/__init__.py | 3 +- askbot/deployment/deployables/base.py | 20 +++++++++++++ askbot/deployment/deployables/components.py | 5 +++- askbot/deployment/deployables/objects.py | 33 ++++++++++----------- askbot/tests/test_installer.py | 16 ++++++++++ 5 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 askbot/deployment/deployables/base.py diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py index afa06252a7..fa1db8749a 100644 --- a/askbot/deployment/deployables/__init__.py +++ b/askbot/deployment/deployables/__init__.py @@ -6,7 +6,8 @@ did up to the error is undone. Yet, this code has no means to undo anything. """ -from .objects import RenderedFile, CopiedFile, EmptyFile, Directory, LinkedDir, AskbotDeploymentError +from .base import AskbotDeploymentError +from .objects import RenderedFile, CopiedFile, EmptyFile, Directory, LinkedDir from .components import AskbotApp, AskbotSite, ProjectRoot diff --git a/askbot/deployment/deployables/base.py b/askbot/deployment/deployables/base.py new file mode 100644 index 0000000000..e1777cb104 --- /dev/null +++ b/askbot/deployment/deployables/base.py @@ -0,0 +1,20 @@ +class AskbotDeploymentError(Exception): + """Use this when something goes wrong while deploying Askbot""" + + +class ObjectWithOutput(object): + def __init__(self, verbosity=1, force=False): + self._verbosity = verbosity + self._force = force + + @property + def verbosity(self): + return self._verbosity + + @verbosity.setter + def verbosity(self, value): + self._verbosity = value + + def print(self, message, verbosity=1): + if verbosity <= self.verbosity: + print(message) diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py index 6921f3994c..cd466c2990 100644 --- a/askbot/deployment/deployables/components.py +++ b/askbot/deployment/deployables/components.py @@ -1,7 +1,8 @@ from askbot.deployment.deployables.objects import * +from askbot.deployment.deployables.base import ObjectWithOutput -class DeployableComponent(object): +class DeployableComponent(ObjectWithOutput): """These constitute sensible deployment chunks of Askbot. For instance, one would never deploy just a settings.py, because Django projects need additional files as well.""" @@ -9,6 +10,7 @@ class DeployableComponent(object): contents = dict() def __init__(self, name=None, contents=None): + super(DeployableComponent, self).__init__(verbosity=2) name = name if name is not None else self.__class__.default_name contents = contents if contents is not None else self.__class__.contents @@ -42,6 +44,7 @@ def _root_deployment_tree(self, tree): root.context = self.context root.forced_overwrite = self.forced_overwrite root.skip_silently = self.skip_silently + root.verbosity = self.verbosity return root @property diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index 69d90a3fe4..73a9872b20 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -2,28 +2,17 @@ import shutil from askbot.deployment.messages import print_message from askbot.deployment.template_loader import DeploymentTemplate +from askbot.deployment.deployables.base import AskbotDeploymentError, ObjectWithOutput -class AskbotDeploymentError(Exception): - """Use this when something goes wrong while deploying Askbot""" - - -class DeployObject(object): +class DeployObject(ObjectWithOutput): """Base class for filesystem objects, i.e. files and directories, that can be/are deployed using askbot-setup.""" def __init__(self, name, src_path=None, dst_path=None): - self._verbosity = 2 + super(DeployObject, self).__init__(verbosity=2) self.name = name self._src_path = src_path self._dst_path = dst_path - @property - def verbosity(self): - return self._verbosity - - @verbosity.setter - def verbosity(self, value): - self._verbosity = value - @property def src_path(self): return self._src_path @@ -55,11 +44,11 @@ def deploy(self): """The main method of this class. DeployableComponents call this method to have this object do the filesystem operations which deploy a single file or directory.""" - print_message(f'* {self.dst} from {self.src}', self.verbosity) + self.print(f'* {self.dst} from {self.src}') try: self._deploy_now() except AskbotDeploymentError as e: - print_message(e, self.verbosity) + self.print(e) class DeployFile(DeployObject): """This class collects all logic w.r.t. writing files. It has to be @@ -97,7 +86,7 @@ def _copy(self): if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. return elif force: - print_message(' ^^^ forced overwrite!', self.verbosity) + self.print(' ^^^ forced overwrite!') else: raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') shutil.copy(self.src, self.dst) @@ -122,6 +111,16 @@ def __init__(self, name, parent=None, *content): if parent is not None: self.dst_path = self.__clean_directory(parent) + @property + def verbosity(self): + return super().verbosity + + @verbosity.setter + def verbosity(self, value): + self._verbosity = value + for child in self.content: + child.verbosity = value + @property def src_path(self): return super().src_path diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 78e0e1c17c..981f30f970 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -46,16 +46,20 @@ def test_individualCopiedFile(self): basename = os.path.basename(self.text_file.name) test = CopiedFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) test = CopiedFile(basename, self.setup_templates.name + '_source_does_not_exist', self.project_root.name) + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) test = CopiedFile(basename + '_bogus', self.setup_templates.name, self.project_root.name) + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) # this should just work and copy the file test = CopiedFile(basename, self.setup_templates.name, self.project_root.name) + test.verbosity = 0 test.deploy() new_file = os.path.join(self.project_root.name, basename) @@ -75,15 +79,19 @@ def test_individualRenderedFile(self): basename = self.jinja2_target test = RenderedFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) test = RenderedFile(basename, self.setup_templates.name + '_source_does_not_exist', self.project_root.name) + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) test = RenderedFile(basename + '_bogus', self.setup_templates.name, self.project_root.name) + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) test = RenderedFile(basename, self.setup_templates.name, self.project_root.name) + test.verbosity = 0 test.context = {'world': 'World'} test.deploy() @@ -104,10 +112,12 @@ def test_individualEmptyFile(self): basename = os.path.basename(self.text_file.name) test = EmptyFile(basename, self.setup_templates.name, self.project_root.name + '_target_does_not_exist') + test.verbosity = 0 self.assertRaises(FileNotFoundError, test.deploy) # this should just work and copy the file test = EmptyFile(basename, self.setup_templates.name, self.project_root.name) + test.verbosity = 0 test.deploy() new_file = os.path.join(self.project_root.name, basename) @@ -125,6 +135,7 @@ def test_individualDirectory(self): basename = 'foobar' test = Directory(basename, self.project_root.name) + test.verbosity = 0 test.deploy() new_dir = os.path.join(self.project_root.name, basename) @@ -191,6 +202,7 @@ def tearDown(self): def test_ProjectRoot(self): test = ProjectRoot(self.project_root.name) test.src_dir = self.setup_templates.name + test.verbosity = 0 test.deploy() comp = self.deployableComponents[test.name] @@ -201,6 +213,7 @@ def test_AskbotSite(self): test = AskbotSite() test.src_dir = self.setup_templates.name test.dst_dir = self.project_root.name + test.verbosity = 0 test.deploy() comp = self.deployableComponents[test.name] @@ -211,6 +224,7 @@ def test_AskbotApp(self): test = AskbotApp() test.src_dir = self.setup_templates.name test.dst_dir = self.project_root.name + test.verbosity = 0 test.deploy() comp = self.deployableComponents[test.name] @@ -226,6 +240,7 @@ def test_addFileBeforeDeploy(self): test.contents.update({'additional.file': CopiedFile}) test.src_dir = self.setup_templates.name + test.verbosity = 0 test.deploy() comp = self.deployableComponents[test.name] @@ -242,6 +257,7 @@ def test_addDirBeforeDeploy(self): test.contents.update({'additionalDir': {'additional.file': CopiedFile}}) test.src_dir = self.setup_templates.name + test.verbosity = 0 test.deploy() comp = self.deployableComponents[test.name] From 280a438e5395720a927c94666c2ae22d6c5e76bc Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 15 Aug 2019 21:07:22 +0200 Subject: [PATCH 32/45] Added implementation for LinkedDir; refined unittests --- askbot/deployment/deployables/objects.py | 12 ++++++- askbot/tests/test_installer.py | 40 +++++++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index 73a9872b20..2888f53c40 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -171,6 +171,15 @@ def forced_overwrite(self, value): for child in self.content: # in practice, we only need this for directories and copied files. child.forced_overwrite = value + def _link_dir(self): + """Derived from __create_path()""" + if os.path.isdir(self.dst): + return + elif os.path.exists(self.dst): + raise AskbotDeploymentError('expect directory or a non-existing path') + else: + os.symlink(self.src, self.dst) + ##################### ## from path_utils ## ##################### @@ -228,4 +237,5 @@ class Directory(DeployDir): pass class LinkedDir(DeployDir): - pass + def _deploy_now(self): + self._link_dir() diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 981f30f970..b89102f281 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -152,7 +152,8 @@ class DeployableComponentsTest(AskbotTestCase): def _flatten_components(self, components): found = [i for i in components if i[1] is RenderedFile - or i[1] is CopiedFile] + or i[1] is CopiedFile + or i[1] is LinkedDir] for descent in [list(i[1].items()) for i in components if isinstance(i[1], dict)]: @@ -194,6 +195,9 @@ def setUp(self): with open(os.path.join( self.setup_templates.name, fname), 'wb') as f: f.write(self.hello.encode('utf-8')) + elif ftype is LinkedDir: + os.makedirs(os.path.join( + self.setup_templates.name, fname), exist_ok=True) def tearDown(self): del self.project_root @@ -206,8 +210,15 @@ def test_ProjectRoot(self): test.deploy() comp = self.deployableComponents[test.name] + comp = self.deployableComponents[test.name] + root_path = self.project_root.name + message = f"\n{root_path}" + message += ' exists' if os.path.isdir(root_path) else ' does not exist' for name, value in comp.contents.items(): - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) + name_path = os.path.join(self.project_root.name, name) + message += f"\n{name_path}" + message += ' exists' if os.path.isdir(name_path) else ' does not exist' + self.assertTrue(os.path.exists(name_path), message) def test_AskbotSite(self): test = AskbotSite() @@ -217,8 +228,17 @@ def test_AskbotSite(self): test.deploy() comp = self.deployableComponents[test.name] + root_path = self.project_root.name + comp_path = os.path.join(root_path, comp.name) + message = f"\n{root_path}" + message += ' exists' if os.path.isdir(root_path) else ' does not exist' + message += f"\n{comp_path}" + message += ' exists' if os.path.isdir(comp_path) else ' does not exist' for name, value in comp.contents.items(): - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, comp.name, name))) + name_path = os.path.join(self.project_root.name, comp.name, name) + message += f"\n{name_path}" + message += ' exists' if os.path.isdir(name_path) else ' does not exist' + self.assertTrue(os.path.exists(name_path),message) def test_AskbotApp(self): test = AskbotApp() @@ -228,10 +248,20 @@ def test_AskbotApp(self): test.deploy() comp = self.deployableComponents[test.name] + root_path = self.project_root.name + comp_path = os.path.join(root_path, comp.name) + message = f"\n{root_path}" + message += ' exists' if os.path.isdir(root_path) else ' does not exist' + message += f"\n{comp_path}" + message += ' exists' if os.path.isdir(comp_path) else ' does not exist' for name, value in comp.contents.items(): - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, comp.name, name))) + name_path = os.path.join(self.project_root.name, comp.name, name) + message += f"\n{name_path}" + message += ' exists' if os.path.isdir(name_path) else ' does not exist' + self.assertTrue(os.path.exists(name_path), message) + - def test_addFileBeforeDeploy(self): +def test_addFileBeforeDeploy(self): test = ProjectRoot(self.project_root.name) another_file = os.path.join(self.setup_templates.name, 'additional.file') From a7ccb60d3c4f29bbc01979ab58ff42634f2e8e14 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 15 Aug 2019 21:47:08 +0200 Subject: [PATCH 33/45] Implemented ProjectRoot.deploy to mimic the actual/current installer --- askbot/deployment/deployables/components.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py index cd466c2990..45c31ac05e 100644 --- a/askbot/deployment/deployables/components.py +++ b/askbot/deployment/deployables/components.py @@ -101,3 +101,14 @@ def __init__(self, install_path): dirname, basename = os.path.split(dirname) super(ProjectRoot, self).__init__(basename) self.dst_dir = dirname + + def deploy(self): + tree = self._grow_deployment_tree(self.contents) + root = self._root_deployment_tree(tree) + # doc has no template in setup_templates. we point src_dir to the + # correct directory after applying all defaults in _root_deployment_tree() + doc = [ node for node in root.content + if isinstance(node,LinkedDir) + and node.name == 'doc' ][0] + doc.src_path = os.path.dirname(doc.src_path) + root.deploy() From cef90cc5386b961113ad10fe7e2a0102404a82ef Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 15 Aug 2019 21:54:25 +0200 Subject: [PATCH 34/45] Fixed indent error --- askbot/tests/test_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index b89102f281..73881e9aa8 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -261,7 +261,7 @@ def test_AskbotApp(self): self.assertTrue(os.path.exists(name_path), message) -def test_addFileBeforeDeploy(self): + def test_addFileBeforeDeploy(self): test = ProjectRoot(self.project_root.name) another_file = os.path.join(self.setup_templates.name, 'additional.file') From 45a8d00b0177a4e1d892c4f32d3ef2b50ffa80b9 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 15 Aug 2019 22:21:13 +0200 Subject: [PATCH 35/45] Deleted unused files --- askbot/deployment/assertions.py | 18 ------------------ askbot/deployment/dev_django.py | 15 --------------- askbot/deployment/package_utils.py | 12 ------------ 3 files changed, 45 deletions(-) delete mode 100644 askbot/deployment/assertions.py delete mode 100644 askbot/deployment/dev_django.py delete mode 100644 askbot/deployment/package_utils.py diff --git a/askbot/deployment/assertions.py b/askbot/deployment/assertions.py deleted file mode 100644 index bc52c7d63d..0000000000 --- a/askbot/deployment/assertions.py +++ /dev/null @@ -1,18 +0,0 @@ -"""assertions regarding deployment of askbot -todo: move here stuff from startup_procedures.py - -the reason - some assertions need to be run in askbot/__init__ -as opposed to startup_procedures.py - which are executed in the -beginning of the models module -""" -from askbot.deployment import package_utils -from askbot.exceptions import DeploymentError - -def assert_package_compatibility(): - """raises an exception if any known incompatibilities - are found - """ - (django_major, django_minor, django_micro) = \ - package_utils.get_django_version() - if django_major < 1: - raise DeploymentError('Django version < 1.0 is not supported by askbot') diff --git a/askbot/deployment/dev_django.py b/askbot/deployment/dev_django.py deleted file mode 100644 index df7d1a5126..0000000000 --- a/askbot/deployment/dev_django.py +++ /dev/null @@ -1,15 +0,0 @@ -from askbot.deployment.deployables import * - -if __name__ == '__main__': - test = CopiedFile('testfile', '/tmp/foo', '/tmp/bar') - test.deploy() - test = RenderedFile('testfile01', '/tmp/foo', '/tmp/bar') - test.context = {'x': 'World'} - test.deploy() - test = Directory('baz', '/tmp') - test.deploy() - askbot_site = AskbotSite() - askbot_app = AskbotApp() - project_root = ProjectRoot('/tmp/project_root_install_dir') - project_root.src_dir = '/tmp/foo' - project_root.deploy() diff --git a/askbot/deployment/package_utils.py b/askbot/deployment/package_utils.py deleted file mode 100644 index 48a469ebf3..0000000000 --- a/askbot/deployment/package_utils.py +++ /dev/null @@ -1,12 +0,0 @@ -"""utilities that determine versions of packages -that are part of askbot - -versions of all packages are normalized to three-tuples -of integers (missing zeroes added) -""" -import django - -def get_django_version(): - """returns three-tuple for the version - of django""" - return django.VERSION[:3] From 1b96f5e42ebbfcd93842b93a2ca4b882beb2d275 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 17:50:05 +0200 Subject: [PATCH 36/45] Common ObjectwithOutput and deploy container files * Both branches had their own definition of that class. * They are using the same single definition now. * Some code cleanup w.r.t. properties and setters * Augmented deployable components to include the files used for by Askbot containers --- .../{deployables => common}/base.py | 0 askbot/deployment/deployables/__init__.py | 2 +- askbot/deployment/deployables/components.py | 19 +++++----- askbot/deployment/deployables/objects.py | 36 +++++++------------ askbot/deployment/parameters/base.py | 20 ++--------- 5 files changed, 28 insertions(+), 49 deletions(-) rename askbot/deployment/{deployables => common}/base.py (100%) diff --git a/askbot/deployment/deployables/base.py b/askbot/deployment/common/base.py similarity index 100% rename from askbot/deployment/deployables/base.py rename to askbot/deployment/common/base.py diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py index fa1db8749a..3315b6e4c6 100644 --- a/askbot/deployment/deployables/__init__.py +++ b/askbot/deployment/deployables/__init__.py @@ -6,7 +6,7 @@ did up to the error is undone. Yet, this code has no means to undo anything. """ -from .base import AskbotDeploymentError +from askbot.deployment.common.base import AskbotDeploymentError from .objects import RenderedFile, CopiedFile, EmptyFile, Directory, LinkedDir from .components import AskbotApp, AskbotSite, ProjectRoot diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py index 45c31ac05e..d7ec4b49ba 100644 --- a/askbot/deployment/deployables/components.py +++ b/askbot/deployment/deployables/components.py @@ -1,6 +1,5 @@ - from askbot.deployment.deployables.objects import * -from askbot.deployment.deployables.base import ObjectWithOutput +from askbot.deployment.common.base import ObjectWithOutput class DeployableComponent(ObjectWithOutput): """These constitute sensible deployment chunks of Askbot. For instance, @@ -28,11 +27,8 @@ def _grow_deployment_tree(self, component): if isinstance(deployable,dict): branch = self._grow_deployment_tree(deployable) todo.append( - Directory( - name, - None, - *branch - )) + Directory(name, None, *branch) + ) else: todo.append(deployable(name)) return todo @@ -81,9 +77,15 @@ class AskbotSite(DeployableComponent): 'django.wsgi': CopiedFile, } +# The naming is terribly misleading here. This is just some of the stuff we use +# for building Askbot containers. This has nothing to do with Django. class AskbotApp(DeployableComponent): default_name = 'askbot_app' - contents = {} + contents = { + 'prestart.sh': CopiedFile, + 'prestart.py': CopiedFile, + 'uwsgi.ini': RenderedFile, # askbot_site, askbot_app + } class ProjectRoot(DeployableComponent): contents = { @@ -93,6 +95,7 @@ class ProjectRoot(DeployableComponent): }, 'doc': LinkedDir, 'upfiles': {}, + 'static': {}, } def __init__(self, install_path): diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index 2888f53c40..0f18c5b21a 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -2,7 +2,7 @@ import shutil from askbot.deployment.messages import print_message from askbot.deployment.template_loader import DeploymentTemplate -from askbot.deployment.deployables.base import AskbotDeploymentError, ObjectWithOutput +from askbot.deployment.common.base import AskbotDeploymentError, ObjectWithOutput class DeployObject(ObjectWithOutput): """Base class for filesystem objects, i.e. files and directories, that can @@ -42,8 +42,8 @@ def _deploy_now(self): def deploy(self): """The main method of this class. DeployableComponents call this method - to have this object do the filesystem operations which deploy a single - file or directory.""" + to have this object do the filesystem operations which deploys + whatever this class represents.""" self.print(f'* {self.dst} from {self.src}') try: self._deploy_now() @@ -51,10 +51,13 @@ def deploy(self): self.print(e) class DeployFile(DeployObject): - """This class collects all logic w.r.t. writing files. It has to be + """This class collects all logic w.r.t. a single files. It has to be subclassed and the subclasses must/should overwrite _deploy_now to call one of the methods defined in this class. At the time of this writing, this - is either _render_with_jinja2 or _copy.""" + is either _render_with_jinja2 or _copy. + + The subclasses may then be used to deploy a single file. + """ def __init__(self, name, src_path=None, dst_path=None): super(DeployFile, self).__init__(name, src_path, dst_path) self.context = dict() @@ -97,11 +100,10 @@ def _copy(self): def _touch(self, times=None): """implementation of unix ``touch`` in python""" #http://stackoverflow.com/questions/1158076/implement-touch-using-python - fhandle = open(self.dst, 'a') try: os.utime(self.dst, times) - finally: - fhandle.close() + except: + open(self.dst, 'a').close() class DeployDir(DeployObject): @@ -111,32 +113,20 @@ def __init__(self, name, parent=None, *content): if parent is not None: self.dst_path = self.__clean_directory(parent) - @property - def verbosity(self): - return super().verbosity - - @verbosity.setter + @DeployObject.verbosity.setter def verbosity(self, value): self._verbosity = value for child in self.content: child.verbosity = value - @property - def src_path(self): - return super().src_path - - @src_path.setter + @DeployObject.src_path.setter def src_path(self, value): value = self.__clean_directory(value) self._src_path = value for child in self.content: child.src_path = value - @property - def dst_path(self): - return super().dst_path - - @dst_path.setter + @DeployObject.dst_path.setter def dst_path(self, value): value = self.__clean_directory(value) self._dst_path = value diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/parameters/base.py index 1ca0bd6aae..f6332d9f0d 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/parameters/base.py @@ -1,13 +1,5 @@ from askbot.utils import console - -class ObjectWithOutput(object): - def __init__(self, verbosity=1, force=False): - self.verbosity = verbosity - self.force = force - - def print(self, message, verbosity=1): - if verbosity <= self.verbosity: - print(message) +from askbot.deployment.common.base import ObjectWithOutput class ConfigManager(ObjectWithOutput): """ConfigManagers are used to ensure the installation can proceed. @@ -42,13 +34,11 @@ class ConfigManager(ObjectWithOutput): } def __init__(self, interactive=True, verbosity=1, force=False): - self._verbosity = verbosity self._interactive = interactive - self._force = force self._catalog = dict() self.keys = set() self._managed_config = dict() - super(ConfigManager, self).__init__(verbosity=verbosity) + super(ConfigManager, self).__init__(verbosity=verbosity, force=force) self.interactive = interactive @property @@ -73,11 +63,7 @@ def force(self, force): if hasattr(handler,'force'): handler.force = force - @property - def verbosity(self): - return self._verbosity - - @verbosity.setter + @ObjectWithOutput.verbosity.setter def verbosity(self, verbosity): self._verbosity = verbosity for name, handler in self._catalog.items(): From 8c59c6ae625bead25c6a046b20e09fb0e7e71469 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 20:09:39 +0200 Subject: [PATCH 37/45] setup files for container installations --- askbot/setup_templates/cron-askbot.sh | 4 ++ askbot/setup_templates/crontab.jinja2 | 1 + askbot/setup_templates/prestart.py | 96 +++++++++++++++++++++++++ askbot/setup_templates/prestart.sh | 10 +++ askbot/setup_templates/uwsgi.ini.jinja2 | 4 ++ 5 files changed, 115 insertions(+) create mode 100755 askbot/setup_templates/cron-askbot.sh create mode 100644 askbot/setup_templates/crontab.jinja2 create mode 100644 askbot/setup_templates/prestart.py create mode 100644 askbot/setup_templates/prestart.sh create mode 100644 askbot/setup_templates/uwsgi.ini.jinja2 diff --git a/askbot/setup_templates/cron-askbot.sh b/askbot/setup_templates/cron-askbot.sh new file mode 100755 index 0000000000..cac229692c --- /dev/null +++ b/askbot/setup_templates/cron-askbot.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +export $(cat /cron_environ | xargs) +cd ${ASKBOT_SITE:-/askbot-site} +/usr/local/bin/python manage.py send_email_alerts > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/askbot/setup_templates/crontab.jinja2 b/askbot/setup_templates/crontab.jinja2 new file mode 100644 index 0000000000..b503a37b65 --- /dev/null +++ b/askbot/setup_templates/crontab.jinja2 @@ -0,0 +1 @@ +0 * * * * /{{ askbot_site }}/{{ askbot_app }}/cron-askbot.sh diff --git a/askbot/setup_templates/prestart.py b/askbot/setup_templates/prestart.py new file mode 100644 index 0000000000..631380b6d2 --- /dev/null +++ b/askbot/setup_templates/prestart.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +import os +import sys + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'askbot_app.settings') + +project_dir = os.environ.get('ASKBOT_SITE') +os.chdir(project_dir) +if project_dir not in sys.path: + sys.path = [ '', project_dir ] + sys.path[1:] + +import askbot as _test_import_of_askbot + +dsn = os.environ.get('DATABASE_URL', None) +admin_to_be_id = 1 + +do_migrate = True +do_admin = True +do_make_admin = True +do_uwsgi_ini = True + +from time import sleep + +if dsn is not None and dsn.startswith('postgres'): + import psycopg2 + print(f'Waiting for database {dsn}') + for retry in range(1,10): + try: + d = psycopg2.connect(dsn) # i hope this takes its time ... + except psycopg2.OperationalError as e: # DB not ready + print(e) + print('Database connected. Peeking...') + c = d.cursor() + c.execute("SELECT * FROM information_schema.tables WHERE table_name='askbot_post';") + r = c.fetchone() + if r is not None: + print('Will not migrate') + do_migrate = False + d.rollback() + try: + c.execute("SELECT * FROM auth_user WHERE username='admin'") + r = c.fetchone() + if r is not None: + print('Will not create admin user') + do_admin = False + except psycopg2.errors.UndefinedTable: + pass + d.rollback() + try: + c.execute(f"SELECT 1 FROM auth_user WHERE id='{admin_to_be_id}' AND is_superuser='yes';") + r = c.fetchone() + if r is not None: + print('Will not make user admin superuser') + do_make_admin = False + except psycopg2.errors.UndefinedTable: + pass + c.close() + d.close() + +from django.core.management import execute_from_command_line + +if do_migrate is True : + print('Running migrate') + argv = [ 'manage.py', 'migrate', '--no-input' ] + execute_from_command_line(argv) + +if do_admin is True and os.environ.get('ADMIN_PASSWORD') is not None: + print('Adding admin user') + argv = [ 'manage.py', 'askbot_add_user', + '--user-name', 'admin', + '--email', 'admin@localhost', + '--password', os.environ.get('ADMIN_PASSWORD'), + '--status', 'd' ] + execute_from_command_line(argv) + +if do_make_admin is True: + print('Grant superuser to admin') + argv = [ 'manage.py', 'add_admin', str(admin_to_be_id), '--noinput' ] + execute_from_command_line(argv) + +if do_uwsgi_ini is True: + print('Preparing uwsgi.ini') + with open(os.path.join(project_dir,'askbot_app','uwsgi.ini'), 'a') as ini: + ini.write("\n") + for rule in [ f'pythonpath = {p}' for p in sys.path]: + ini.write(rule + "\n") + +if os.environ.get('NO_CRON') is None: + print('Preparing cron_environ') + with open('/cron_environ', 'w+') as cron_env: + for var in [ f'{name}="{os.environ.get(name)}"' + for name in [ 'PATH', 'DATABASE_URL', 'SECRET_KEY', + 'PYTHONUNBUFFERED', 'ASKBOT_SITE' ]]: + cron_env.write(var + "\n") + cron_env.write("PYTHONPATH='{}'\n".format(str.join(":", sys.argv))) diff --git a/askbot/setup_templates/prestart.sh b/askbot/setup_templates/prestart.sh new file mode 100644 index 0000000000..da16d5bc5e --- /dev/null +++ b/askbot/setup_templates/prestart.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [ -x /wait ]; then /wait; else sleep 64; fi + +python ${ASKBOT_SITE}/askbot_app/prestart.py + +if [ -z "$NO_CRON" ]; then + crond || cron +fi + diff --git a/askbot/setup_templates/uwsgi.ini.jinja2 b/askbot/setup_templates/uwsgi.ini.jinja2 new file mode 100644 index 0000000000..36bed031e8 --- /dev/null +++ b/askbot/setup_templates/uwsgi.ini.jinja2 @@ -0,0 +1,4 @@ +[uwsgi] +static-map = /m=/{{ askbot_site }}/static +env = DJANGO_SETTINGS_MODULE={{ askbot_app }}.settings +wsgi-file = /{{ askbot_site }}/{{ askbot_app }}/django.wsgi From 5bed3d99b9d4046073d1dea1c9cd1f38322aecac Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 20:10:32 +0200 Subject: [PATCH 38/45] Moved hardcoded value from installer into template * there is no reason to render the information into the file if it can't ever change --- askbot/setup_templates/settings.py.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askbot/setup_templates/settings.py.jinja2 b/askbot/setup_templates/settings.py.jinja2 index 2f39ccc8e1..a87961e05a 100644 --- a/askbot/setup_templates/settings.py.jinja2 +++ b/askbot/setup_templates/settings.py.jinja2 @@ -189,7 +189,7 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.sitemaps', 'django.contrib.sites', - {{ staticfiles_app }} + 'django.contrib.staticfiles', ## extra packages 'avatar', From 3ddbd71b2e7ce9767e8a73424abcefc5a0b71597 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 20:12:22 +0200 Subject: [PATCH 39/45] New merged installer * AskbotInstaller is now also derived from ObjectWithOutput * uses parameters module for input validation * uses deployables module for deployment * big changes to askbot.deployment.path_utils as its functionality was copied/moved/delegated into new modules * small improvements to input validing classes --- askbot/deployment/__init__.py | 326 +++++++-------------- askbot/deployment/parameters/__init__.py | 5 + askbot/deployment/parameters/database.py | 16 + askbot/deployment/parameters/filesystem.py | 2 +- askbot/deployment/path_utils.py | 88 +----- 5 files changed, 136 insertions(+), 301 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 8b554e990a..f2911d8482 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -12,18 +12,17 @@ from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate from askbot.deployment.parameters import ConfigManagerCollection +from askbot.deployment.common.base import ObjectWithOutput +from askbot.deployment.deployables.components import DeployableComponent import askbot.deployment.deployables as deployable -import shutil -class AskbotSetup: - - PROJECT_FILES_TO_CREATE = {'manage.py'} - APP_FILES_TO_CREATE = set(path_utils.FILES_TO_CREATE) - {'manage.py'} - SOURCE_DIR = os.path.dirname(os.path.dirname(__file__)) # a.k.a. ASKBOT_ROOT in settings.py +class AskbotSetup(ObjectWithOutput): + ASKBOT_ROOT = os.path.dirname(os.path.dirname(__file__)) + SOURCE_DIR = 'setup_templates' def __init__(self, interactive=True, verbosity=-128): + super(AskbotSetup, self).__init__(verbosity=verbosity) self.parser = ArgumentParser(description="Setup a Django project and app for Askbot") - self.verbosity = verbosity self._todo = {} self.configManagers = ConfigManagerCollection(interactive=interactive, verbosity=verbosity) self.database_engines = self.configManagers.configManager( @@ -71,8 +70,9 @@ def _add_settings_args(self): def _add_setup_args(self): """Control the behaviour of this setup procedure Adds - --create - project + --create-project --dir-name, -n + --app-name --verbose, -v --force --dry-run @@ -97,7 +97,7 @@ def _add_setup_args(self): self.parser.add_argument( "--app-name", dest="app_name", - default='askbot_app', + default=None, help="Django app name (subdir) for this Askbot deployment in the target Django project." ) @@ -237,242 +237,130 @@ def _add_db_args(self): help = "The database port" ) - def _set_verbosity(self, v): - self.verbosity = v + @ObjectWithOutput.verbosity.setter + def verbosity(self, v): + self._verbosity = v self.configManagers.verbosity = v - # I think this logic can be immediately attached to argparse - # it would be a hack though - def _set_create_project(self, options): - # Currently the --create-project option only changes the installer's - # behaviour if one passes "container-uwsgi" as argument - todo = [ 'django' ] # This is the default as Askbot has always worked - wish = str.lower(options['create_project']) - if wish in [ 'no', 'none', 'false', '0']: - todo = [ 'nothing' ] - elif wish == 'container-uwsgi': - todo.append(wish) - self._todo['create_project'] = todo + def _process_args(self, options): + """ + In this method we fiddle with askbot-setup parameters, i.e. cli + arguments. This is run BEFORE the installer uses the ConfigManagers to + do sanity checks and interact with the user. + """ + options = vars(options) # use dictionary instead of Namespace + # force + force = False # effectively disables the --force switch! + # logdir + logdir_name = path_utils.LOG_DIR_NAME # will become a parameter soon, me thinks + # secret_key + secret_key = generate_random_key() + if options['no_secret_key']: + secret_key = '' + # app_name + app_name = options['app_name'] + if app_name is None: + app_name = os.path.basename(options['dir_name']) + options['force'] = force + options['logdir_name'] = logdir_name + options['secret_key'] = secret_key + options['app_name'] = app_name + options['create_project'] = str.lower(options['create_project']) + return options def __call__(self): # this is the main part of the original askbot_setup() try: + self.print(messages.DEPLOY_PREAMBLE) + options = self.parser.parse_args() - options = vars(options) - options['force'] = False # disable the --force switch! - self._set_verbosity(options['verbosity']) - self._set_create_project(options) - print_message(messages.DEPLOY_PREAMBLE, self.verbosity) + options = self._process_args(options) + self.verbosity = options['verbosity'] + self.configManagers.complete(options) + #database_interface = [ e[1] for e in self.database_engines + # if e[0] == options['database_engine'] ][0] + #options['database_engine'] = database_interface - options['dir_name'] = path_utils.clean_directory(options['dir_name']) - options['secret_key'] = '' if options['no_secret_key'] else generate_random_key() + nothing = DeployableComponent() + nothing.deploy = lambda: None - self.configManagers.complete(options) + project = deployable.ProjectRoot(options['dir_name']) + project.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + project.contents.update({ + options['logdir_name']: {options['logfile_name']: deployable.EmptyFile} + }) + + site = deployable.AskbotSite(options['app_name']) + site.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + site.dst_dir = options['dir_name'] + site.context.update(options) + + uwsgi = deployable.AskbotApp() + uwsgi.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + uwsgi.dst_dir = options['dir_name'] + uwsgi.context.update({ + 'askbot_site': options['dir_name'], + 'askbot_app': uwsgi.name, # defaults to askbot_app + }) - database_interface = [ e[1] for e in self.database_engines - if e[0] == options['database_engine'] ][0] - options['database_engine'] = database_interface + project.context = {'settings_path': f'{site.name}.settings'} + + todo = [ project, site ] + + if options['create_project'] in ['no', 'none', 'false', '0', 'nothing']: + todo = [ nothing ] + elif options['create_project'] == 'container-uwsgi': + project.contents.update({ + 'cron': { + 'crontab': deployable.RenderedFile, # askbot_site, askbot_app + 'cron-askbot.sh': deployable.CopiedFile, + }}) + todo.append(uwsgi) if options['dry_run']: - from pprint import pformat - print_message(pformat(options), self.verbosity) - print_message(pformat(self.__dict__), self.verbosity) - raise KeyboardInterrupt + raise StopIteration + + for component in todo: + component.deploy() + + help_file = path_utils.get_path_to_help_file() - self.deploy_askbot_new(options) + self.print(messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}) - if database_interface == 'postgresql_psycopg2': + if options['database_engine'] == 'postgresql_psycopg2': try: import psycopg2 except ImportError: - print_message('\nNEXT STEPS: install python binding for postgresql', self.verbosity) - print_message('pip install psycopg2-binary\n', self.verbosity) - elif database_interface == 'mysql': + self.print('\nNEXT STEPS: install python binding for postgresql') + self.print('pip install psycopg2-binary\n') + elif options['database_engine'] == 'mysql': try: import _mysql except ImportError: - print_message('\nNEXT STEP: install python binding for mysql', self.verbosity) - print_message('pip install mysql-python\n', self.verbosity) + self.print('\nNEXT STEP: install python binding for mysql') + self.print('pip install mysql-python\n') except KeyboardInterrupt: - print_message("\n\nAborted.", self.verbosity) + self.print("\n\nAborted.") sys.exit(1) - def _install_copy(self, copy_list, forced_overwrite=[], skip_silently=[]): - print_message('Copying files:', self.verbosity) - for src,dst in copy_list: - print_message(f'* to {dst} from {src}', self.verbosity) - if not os.path.exists(dst): - shutil.copy(src, dst) - continue - matches = [ dst for c in forced_overwrite - if dst.endswith(f'{os.path.sep}{c}') ] - if len(matches) > 0: - print_message(' ^^^ forced overwrite!', self.verbosity) - shutil.copy(src, dst) - elif dst.split(os.path.sep)[-1] not in skip_silently: - print_message(f' ^^^ you already have one, please add contents of {src}', self.verbosity) - print_message('Done.', self.verbosity) - - def _install_render_with_jinja2(self, render_list, context): - print_message('Rendering files:', self.verbosity) - template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended - for src, dst in render_list: - if os.path.exists(dst): - print_message(f'* you already have a file "{dst}" please merge the contents', self.verbosity) - continue - print_message(f'* {dst} from {src}', self.verbosity) - template.tmpl_path = src - output = template.render(context) - with open(dst, 'w+') as output_file: - output_file.write(output) - print_message('Done.', self.verbosity) - - def _create_new_django_project(self, install_dir, options): - log_dir = os.path.join(install_dir, path_utils.LOG_DIR_NAME) - log_file = os.path.join(log_dir, options['logfile_name']) - - create_me = [ install_dir, log_dir ] - copy_me = list() - - if 'django' in self._todo.get('create_project',[]): - src = lambda x:os.path.join(self.SOURCE_DIR, 'setup_templates', x) - dst = lambda x:os.path.join(install_dir, x) - copy_me.extend([ - ( src(file_name), dst(file_name) ) - for file_name in self.PROJECT_FILES_TO_CREATE - ]) - - for d in create_me: - path_utils.create_path(d) - - path_utils.touch(log_file) - self._install_copy(copy_me, skip_silently=path_utils.BLANK_FILES) - - def _create_new_django_app(self, options): - options['askbot_site'] = options['dir_name'] # b/c the jinja template uses askbot_site - options['askbot_app'] = options['app_name'] # b/c the jinja template uses askbot_app - app_dir = os.path.join(options['dir_name'], options['app_name']) - - create_me = [ app_dir ] - copy_me = list() - render_me = list() - - if 'django' in self._todo.get('create_project',[]): - src = lambda x:os.path.join(self.SOURCE_DIR, 'setup_templates', x) - dst = lambda x:os.path.join(app_dir, x) - copy_me.extend([ - ( src(file_name), dst(file_name) ) - for file_name in self.APP_FILES_TO_CREATE - ]) - render_me.extend([ - ( src('settings.py.jinja2'), dst('settings.py') ) - ]) - - if 'container-uwsgi' in self._todo.get('create_project',[]): - src = lambda x:os.path.join(self.SOURCE_DIR, 'container', x) - dst = lambda x:os.path.join(app_dir, x) - copy_me.extend([ - ( src(file_name), dst(file_name) ) - for file_name in [ 'cron-askbot.sh', 'prestart.sh', 'prestart.py' ] - ]) - render_me.extend([ - ( src(file_name), dst(file_name) ) - for file_name in [ 'crontab', 'uwsgi.ini' ] - ]) - - for d in create_me: - path_utils.create_path(d) - - self._install_copy(copy_me, skip_silently=path_utils.BLANK_FILES, - forced_overwrite=['urls.py']) - - self._install_render_with_jinja2(render_me, options) - - if len(options['local_settings']) > 0 \ - and os.path.exists(options['local_settings']): - dst = os.path.join(app_dir, 'settings.py') - print_message(f'Appending {options["local_settings"]} to {dst}', self.verbosity) - with open(dst, 'a') as settings_file: - with open(options['local_settings'], 'r') as local_settings: - settings_file.write('\n') - settings_file.write(local_settings.read()) - print_message('Done.', self.verbosity) - - def deploy_askbot(self, options): - """function that creates django project files, - all the neccessary directories for askbot, - and the log file - """ - - create_new_project = True - if os.path.exists(options['dir_name']) and \ - path_utils.has_existing_django_project(options['dir_name']) and \ - options['force'] is False: - create_new_project = False - - options['staticfiles_app'] = "'django.contrib.staticfiles'," # Fixme: move this into the template - options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' # Fixme: move this into the template - - if create_new_project is True: - self._create_new_django_project(options['dir_name'], options) - - self._create_new_django_app(options) - - help_file = path_utils.get_path_to_help_file() - - if create_new_project: - print_message( - messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}, - self.verbosity - ) - else: - print_message( - messages.HOW_TO_ADD_ASKBOT_TO_DJANGO % {'help_file': help_file}, - self.verbosity - ) - - def deploy_askbot_new(self, options): - create_new_project = True - if os.path.exists(options['dir_name']) and \ - path_utils.has_existing_django_project(options['dir_name']) and \ - options.force is False: - create_new_project = False - - options['staticfiles_app'] = "'django.contrib.staticfiles'," - options['auth_context_processor'] = 'django.contrib.auth.context_processors.auth' - - project = deployable.ProjectRoot(options['dir_name']) - site = deployable.AskbotSite(project.name) - - if create_new_project is True: - project.src_dir = os.path.join(self.SOURCE_DIR, 'setup_templates') - project.contents.update({ - path_utils.LOG_DIR_NAME: {options['logfile_name']: deployable.EmptyFile} - }) - project.context = {'settings_path': f'{site.name}.settings'} - project.deploy() - - site.src_dir = os.path.join(self.SOURCE_DIR, 'setup_templates') - site.dst_dir = options['dir_name'] - site.context.update(options) - site.deploy() - - help_file = path_utils.get_path_to_help_file() - - if create_new_project: - print_message( - messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}, - self.verbosity - ) - else: - print_message( - messages.HOW_TO_ADD_ASKBOT_TO_DJANGO % {'help_file': help_file}, - self.verbosity - ) + except StopIteration: + from pprint import pformat + self.print(pformat(options)) + self.print(pformat(self.__dict__)) + sys.exit(0) + # ToDo: The following is not yet implemented in the new code + #if len(options['local_settings']) > 0 \ + #and os.path.exists(options['local_settings']): + # dst = os.path.join(app_dir, 'settings.py') + # print_message(f'Appending {options["local_settings"]} to {dst}', self.verbosity) + # with open(dst, 'a') as settings_file: + # with open(options['local_settings'], 'r') as local_settings: + # settings_file.write('\n') + # settings_file.write(local_settings.read()) + # print_message('Done.', self.verbosity) -# set to askbot_setup_orig to return to original installer askbot_setup = AskbotSetup(interactive=True, verbosity=1) diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index 477a274e90..ed9ff3b549 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -7,6 +7,11 @@ # ConfigManager because the collection serves a different purpose than the # individual manager, but they are still quite similar class ConfigManagerCollection(ConfigManager): + """ + This is the main config manager that will be used by the Askbot installer. + It is a hard coded ordered collection of all config managers the installer + shall use. + """ def __init__(self, interactive=False, verbosity=0): super(ConfigManagerCollection, self).__init__(interactive=interactive, verbosity=verbosity) self.register('database', DbConfigManager(interactive=interactive, verbosity=verbosity)) diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index 10a5a73a6d..c3abfddcaa 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -51,6 +51,22 @@ def _remember(self, name, value): self._catalog['database_user'].defaultOk = True self._catalog['database_password'].defaultOk = True + def _complete(self, name, current_value): + """ + Wrap the default _complete() to implement a special handling of + `database_engine`. While the user selects database engines by index, + i.e. 1,2,3 or 4, at the time of this writing, the installer and Askbot + use Django module names (I think that's what this is). Therefore we + perform a lookup after the user made their final choice and return + the name, rather than the index. + """ + ret = super(DbConfigManager, self)._complete(name, current_value) + if name == 'database_engine': + return [ e[1] for e in + self.configField(name).database_engines + if e[0] == ret ].pop() + return ret + class DbEngine(ConfigField): defaultOk = False diff --git a/askbot/deployment/parameters/filesystem.py b/askbot/deployment/parameters/filesystem.py index 7155d405d2..8a04d62754 100644 --- a/askbot/deployment/parameters/filesystem.py +++ b/askbot/deployment/parameters/filesystem.py @@ -125,7 +125,7 @@ def ask_user(self, current_value): class AppDirName(BaseDirName): defaultOk = True, - default = 'askbot_app', + default = None, user_prompt = "Please enter a Django App name for this Askbot deployment." def acceptable(self, value): diff --git a/askbot/deployment/path_utils.py b/askbot/deployment/path_utils.py index b8e813cae7..7255015ffe 100644 --- a/askbot/deployment/path_utils.py +++ b/askbot/deployment/path_utils.py @@ -10,12 +10,10 @@ import tempfile import re import glob -import shutil import imp from askbot.deployment import messages from askbot.utils import console -from askbot.deployment.template_loader import DeploymentTemplate as SettingsTemplate FILES_TO_CREATE = ('__init__.py', 'manage.py', 'urls.py', 'django.wsgi', 'celery_app.py') @@ -55,15 +53,14 @@ def clean_directory(directory): return None return directory - +# Only used in can_create_path def directory_is_writable(directory): """returns True if directory exists and is writable, False otherwise """ - tempfile.tempdir = directory try: #run writability test - temp_path = tempfile.mktemp() + temp_path = tempfile.mktemp(dir=directory) assert(os.path.dirname(temp_path) == directory) temp_file = open(temp_path, 'w') temp_file.close() @@ -73,6 +70,7 @@ def directory_is_writable(directory): return False +# Only used in get_install_directory def can_create_path(directory): """returns True if user can write file into directory even if it does not exist yet @@ -120,6 +118,7 @@ def find_parent_dir_with_django(directory): return None +# Only used in get_install_directory def path_is_clean_for_django(directory): """returns False if any of the parent directories contains a Django project, otherwise True @@ -129,6 +128,7 @@ def path_is_clean_for_django(directory): return (django_dir is None) +# Can be removed after fixing askbot.tests.test_installer def create_path(directory): """equivalent to mkdir -p""" if os.path.isdir(directory): @@ -138,94 +138,20 @@ def create_path(directory): else: os.makedirs(directory) -def touch(file_path, times = None): - """implementation of unix ``touch`` in python""" - #http://stackoverflow.com/questions/1158076/implement-touch-using-python - fhandle = open(file_path, 'a') - try: - os.utime(file_path, times) - finally: - fhandle.close() SOURCE_DIR = os.path.dirname(os.path.dirname(__file__)) def get_path_to_help_file(): """returns path to the main plain text help file""" return os.path.join(SOURCE_DIR, 'doc', 'INSTALL') -def deploy_into(install_dir, new_project = False, verbosity = 1, context = None): - """will copy necessary files into the target directory - """ - assert(isinstance(new_project, bool)) - if new_project: - if verbosity >= 1: - print('Copying files: ') - for file_name in FILES_TO_CREATE: - src_file = os.path.join(SOURCE_DIR, 'setup_templates', file_name) - if verbosity >= 1: - print(f'* {file_name}') - if os.path.exists(os.path.join(install_dir, file_name)): - if file_name in BLANK_FILES: - continue - if file_name == 'urls.py' and verbosity >= 1: - print(' ^^^ forced overwrite!') - else: - if verbosity >= 1: - print(f' ^^^ you already have one, please add contents of {src_file}') - continue - shutil.copy(src_file, install_dir) - #create log directory - log_dir = os.path.join(install_dir, LOG_DIR_NAME) - create_path(log_dir) - touch(os.path.join(log_dir, 'askbot.log')) - - #creating settings file from template - if verbosity >= 1: - print("Creating settings file") - settings_contents = SettingsTemplate(context).render() - settings_path = os.path.join(install_dir, 'settings.py') - if os.path.exists(settings_path): - if verbosity >= 1: - print("* you already have a settings file please merge the contents") - else: - with open(settings_path, 'w+') as settings_file: - settings_file.write(settings_contents) - #Grab the file! - if os.path.exists(context['local_settings']): - with open(context['local_settings'], 'r') as local_settings: - settings_file.write('\n') - settings_file.write(local_settings.read()) - - if verbosity >= 1: - print("settings file created") - # end if new_project - if verbosity >= 1: - print('') - app_dir = os.path.join(install_dir, 'askbot') - - if verbosity >= 1: - print('copying directories: ') - copy_dirs = ('doc', 'cron', 'upfiles') - for dir_name in copy_dirs: - src_dir = os.path.join(SOURCE_DIR, dir_name) - dst_dir = os.path.join( app_dir, dir_name) - if os.path.abspath(src_dir) == os.path.abspath(dst_dir): # this is actually just a special form of an already existing directory - continue - if verbosity >= 1: - print(f'* {dir_name}') - if os.path.exists(dst_dir): - if verbosity >= 1: - print(' ^^^ already exists - skipped') - continue - shutil.copytree(src_dir, dst_dir) - if verbosity >= 1: - print('') - +# Only used in get_install_directory def dir_name_unacceptable_for_django_project(directory): dir_name = os.path.basename(directory) if re.match(r'[_a-zA-Z][\w-]*$', dir_name): return False return True +# Only used in get_install_directory def dir_taken_by_python_module(directory): """True if directory is not taken by another python module""" dir_name = os.path.basename(directory) From fded777b5f2b1966a5c40cfcb1aa8b3bfd715124 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 20:15:43 +0200 Subject: [PATCH 40/45] Fix tests and manaagement commands * There were imports of askbot.deployment.path_utils, which was removed somewhere along the line. -> removed imports * Some improvements to existing installer tests --- askbot/management/commands/merge_users.py | 1 - askbot/tests/test_installer.py | 66 ++++++++++++++++------- askbot/tests/test_page_load.py | 1 - 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/askbot/management/commands/merge_users.py b/askbot/management/commands/merge_users.py index 792709df1a..01e0a6684c 100644 --- a/askbot/management/commands/merge_users.py +++ b/askbot/management/commands/merge_users.py @@ -1,7 +1,6 @@ from django.core.management.base import CommandError, BaseCommand from django.db import transaction -from askbot.deployment import package_utils from askbot.models import (User, LocalizedUserProfile, Post, GroupMembership ) diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index d0488eaf23..6094a5662d 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -439,7 +439,7 @@ def test_propagate_attributes(self): self.assertEqual(obj.verbosity, self.installer.verbosity) for new_verbosity in [2, 5, 13, 23, 42, 666, 1337, 16061, self.installer.verbosity]: - self.installer._set_verbosity(new_verbosity) + self.installer.verbosity = new_verbosity for obj in has_verbosity: self.assertEqual(obj.verbosity, new_verbosity) @@ -459,9 +459,10 @@ def test_propagate_attributes(self): self.assertEqual(obj.interactive, new_interactive) def test_flow_dry_run(self): + destdir = tempfile.TemporaryDirectory() self.installer.configManagers.interactive=False # minimal_viable_argument_set - mva = ['--dir-name', '/tmp/AskbotTestDir', + mva = ['--dir-name', destdir.name, '--db-engine', '2', '--db-name', '/tmp/AskbotTest.db', '--cache-engine', '3'] @@ -475,7 +476,8 @@ def test_flow_dry_run(self): self.assertEqual(mock.call_count, 1) def test_flow_skip_deploy(self): - mva = ['--dir-name', '/tmp/AskbotTestDir', + destdir = tempfile.TemporaryDirectory() + mva = ['--dir-name', destdir.name, '--db-engine', '2', '--db-name', '/tmp/AskbotTest.db', '--cache-engine', '3'] @@ -492,7 +494,8 @@ def test_flow_skip_deploy(self): self.fail(f'Running the installer raised {e}') def test_flow_mock_deployment(self): - mva = ['--dir-name', '/tmp/AskbotTestDir', + destdir = tempfile.TemporaryDirectory() + mva = ['--dir-name',destdir.name, '--db-engine', '2', '--db-name', '/tmp/AskbotTest.db', '--cache-engine', '3'] @@ -501,8 +504,8 @@ def test_flow_mock_deployment(self): parse_args = MagicMock(name='parse_args', return_value=opts) self.installer.parser.parse_args = parse_args fake_open = mock_open(read_data='foobar') - with patch('askbot.deployment.path_utils.create_path'), patch('askbot.deployment.path_utils.touch'), patch('shutil.copy'), patch('builtins.open', fake_open): - self.installer() + #with patch('askbot.deployment.path_utils.create_path'), patch('shutil.copy'), patch('builtins.open', fake_open): + self.installer() class DeployObjectsTest(AskbotTestCase): @@ -699,13 +702,28 @@ def tearDown(self): del self.project_root del self.setup_templates + def test_TestSetup(self): + self.assertTrue(os.path.isdir(self.project_root.name)) + self.assertTrue(os.path.isdir(self.setup_templates.name)) + self.assertFalse(self.project_root.name == self.setup_templates.name) + + def test_ProjectRoot(self): + mock_doc_dir = tempfile.TemporaryDirectory() + os.mkdir(os.path.join(mock_doc_dir.name, 'doc')) test = ProjectRoot(self.project_root.name) test.src_dir = self.setup_templates.name test.verbosity = 0 - test.deploy() - comp = self.deployableComponents[test.name] + # ProiectRoot.deploy works under the assumption that 'setup_templates' + # and 'doc' are directories inside the same parent directory. While + # this holds true for an actual Askbot installation, it doesn't in this + # unittest. We use the following patch context to align the + # directory structure assumption of ProjectRoot.deploy with the + # test environment, which is auto-generated temporary directories. + with patch('os.path.dirname', return_value=mock_doc_dir.name): + test.deploy() + comp = self.deployableComponents[test.name] root_path = self.project_root.name message = f"\n{root_path}" @@ -713,9 +731,10 @@ def test_ProjectRoot(self): for name, value in comp.contents.items(): name_path = os.path.join(self.project_root.name, name) message += f"\n{name_path}" - message += ' exists' if os.path.isdir(name_path) else ' does not exist' + message += ' exists' if os.path.exists(name_path) else ' does not exist' self.assertTrue(os.path.exists(name_path), message) + def test_AskbotSite(self): test = AskbotSite() test.src_dir = self.setup_templates.name @@ -758,35 +777,44 @@ def test_AskbotApp(self): def test_addFileBeforeDeploy(self): - test = ProjectRoot(self.project_root.name) - another_file = os.path.join(self.setup_templates.name, 'additional.file') with open(another_file, 'wb') as f: f.write(self.hello.encode('utf-8')) + + test = AskbotSite() test.contents.update({'additional.file': CopiedFile}) test.src_dir = self.setup_templates.name + test.dst_dir = self.project_root.name test.verbosity = 0 test.deploy() + del test.contents['additional.file'] + comp = self.deployableComponents[test.name] - for name, value in comp.contents.items(): - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, 'additional.file'))) + name_path = os.path.join(self.project_root.name, comp.name, 'additional.file') + message = f"\n{name_path}" + message += ' exists' if os.path.isdir(name_path) else ' does not exist' + self.assertTrue(os.path.exists(name_path), message) - def test_addDirBeforeDeploy(self): - test = ProjectRoot(self.project_root.name) + def test_addDirBeforeDeploy(self): another_file = os.path.join(self.setup_templates.name, 'additional.file') with open(another_file, 'wb') as f: f.write(self.hello.encode('utf-8')) + + test = AskbotSite() test.contents.update({'additionalDir': {'additional.file': CopiedFile}}) test.src_dir = self.setup_templates.name + test.dst_dir = self.project_root.name test.verbosity = 0 test.deploy() + del test.contents['additionalDir'] + comp = self.deployableComponents[test.name] - for name, value in comp.contents.items(): - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, name))) - self.assertTrue(os.path.exists(os.path.join(self.project_root.name, 'additionalDir', 'additional.file'))) + name_path = os.path.join(self.project_root.name, comp.name, 'additionalDir', 'additional.file') + message = f"\n{name_path}" + message += ' exists' if os.path.isdir(name_path) else ' does not exist' + self.assertTrue(os.path.exists(name_path), message) diff --git a/askbot/tests/test_page_load.py b/askbot/tests/test_page_load.py index 5c73ac598e..cc8a67d729 100644 --- a/askbot/tests/test_page_load.py +++ b/askbot/tests/test_page_load.py @@ -15,7 +15,6 @@ import askbot from askbot import models from askbot.utils.slug import slugify -from askbot.deployment import package_utils from askbot.tests.utils import AskbotTestCase from askbot.conf import settings as askbot_settings from askbot.tests.utils import skipIf From e5f3b94c5e4caf361cc4a29bd61a3e1c172a42ff Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 3 Oct 2019 20:17:47 +0200 Subject: [PATCH 41/45] adapt tox.ini * does not need to set DJANGO_SETTINGS anymore, because the installer does a better job deployoing the files now --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 47f10a4ada..8544755347 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,6 @@ whitelist_externals = mkdir rm psql -setenv = DJANGO_SETTINGS_MODULE = askbot_app.settings commands_pre = mkdir -p {toxinidir}/deploy_askbot psql -h localhost -p 5432 -U postgres -c "DROP DATABASE IF EXISTS deploy_askbot" @@ -58,7 +57,6 @@ deps = whitelist_externals = mkdir rm -setenv = DJANGO_SETTINGS_MODULE = askbot_app.settings commands_pre = mkdir -p {toxinidir}/deploy_askbot commands = From d1597db5713e416baeadfe15a86bacfce5548b7c Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 10 Oct 2019 22:42:39 +0200 Subject: [PATCH 42/45] Refactor installer [1/3] -- base module * moved all generic logic and base classes into a base module --- askbot/deployment/base/__init__.py | 14 ++ askbot/deployment/base/configfield.py | 33 +++ .../base.py => base/configmanager.py} | 79 +++---- askbot/deployment/base/deployablecomponent.py | 68 ++++++ askbot/deployment/base/deployableobject.py | 209 ++++++++++++++++++ askbot/deployment/base/exceptions.py | 23 ++ .../base.py => base/objectwithoutput.py} | 4 - .../deployment/{ => base}/template_loader.py | 0 8 files changed, 389 insertions(+), 41 deletions(-) create mode 100644 askbot/deployment/base/__init__.py create mode 100644 askbot/deployment/base/configfield.py rename askbot/deployment/{parameters/base.py => base/configmanager.py} (72%) create mode 100644 askbot/deployment/base/deployablecomponent.py create mode 100644 askbot/deployment/base/deployableobject.py create mode 100644 askbot/deployment/base/exceptions.py rename askbot/deployment/{common/base.py => base/objectwithoutput.py} (79%) rename askbot/deployment/{ => base}/template_loader.py (100%) diff --git a/askbot/deployment/base/__init__.py b/askbot/deployment/base/__init__.py new file mode 100644 index 0000000000..70e3eb2cfe --- /dev/null +++ b/askbot/deployment/base/__init__.py @@ -0,0 +1,14 @@ +from .objectwithoutput import ObjectWithOutput +from .deployablecomponent import DeployableComponent +from .deployableobject import DeployableObject, DeployableFile, DeployableDir +from .configfield import ConfigField +from .configmanager import ConfigManager, ConfigManagerCollection + +from askbot.deployment.base import exceptions + + + +__all__ = ['exceptions', 'DeployableComponent', 'ObjectWithOutput', + 'DeployableObject', 'DeployableFile', 'DeployableDir', + 'ConfigField', 'ConfigManager', 'ConfigManagerCollection', + ] diff --git a/askbot/deployment/base/configfield.py b/askbot/deployment/base/configfield.py new file mode 100644 index 0000000000..7f40321bb4 --- /dev/null +++ b/askbot/deployment/base/configfield.py @@ -0,0 +1,33 @@ +from .objectwithoutput import ObjectWithOutput +from askbot.utils import console + +class ConfigField(ObjectWithOutput): + defaultOk = True + default = None + user_prompt = 'Please enter something' + + def __init__(self, defaultOk=None, default=None, user_prompt=None, verbosity=1): + super(ConfigField, self).__init__(verbosity=verbosity) + self.defaultOk = self.__class__.defaultOk if defaultOk is None else defaultOk + self.default = self.__class__.default if default is None else default + self.user_prompt = self.__class__.user_prompt if user_prompt is None else user_prompt + + def acceptable(self, value): + """High level sanity check for a specific value. This method is called + for an installation parameter with the value provided by the user, or + the default value, if the user didn't provide any value. There must be + a boolean response, if the installation can proceed with :var:value as + the setting for this ConfigField.""" + #self.print(f'This is {cls.__name__}.acceptable({value}) {cls.defaultOk}', 2) + if value is None and self.default is None or value == self.default: + return self.defaultOk + return True + + def ask_user(self, current): + """Prompt the user to provide a value for this installation + parameter.""" + user_prompt = self.user_prompt + if self.defaultOk is True: + user_prompt += ' (Just press ENTER, to use the current '\ + + f'value "{current}")' + return console.simple_dialog(user_prompt) diff --git a/askbot/deployment/parameters/base.py b/askbot/deployment/base/configmanager.py similarity index 72% rename from askbot/deployment/parameters/base.py rename to askbot/deployment/base/configmanager.py index f6332d9f0d..aa8834433f 100644 --- a/askbot/deployment/parameters/base.py +++ b/askbot/deployment/base/configmanager.py @@ -1,5 +1,4 @@ -from askbot.utils import console -from askbot.deployment.common.base import ObjectWithOutput +from .objectwithoutput import ObjectWithOutput class ConfigManager(ObjectWithOutput): """ConfigManagers are used to ensure the installation can proceed. @@ -35,9 +34,10 @@ class ConfigManager(ObjectWithOutput): def __init__(self, interactive=True, verbosity=1, force=False): self._interactive = interactive - self._catalog = dict() - self.keys = set() - self._managed_config = dict() + self._catalog = dict() # we use this to hold the ConfigFields + self.keys = set() # we use this for scoping and consistency + self._ordered_keys = list() # we use this for ordering our keys + self._managed_config = dict() # we use this as regestry for completed work super(ConfigManager, self).__init__(verbosity=verbosity, force=force) self.interactive = interactive @@ -75,9 +75,11 @@ def register(self, name, handler): Parameters: - name: the install parameter to handle - handler: the class to handle the parameter""" + handler.verbosity = self.verbosity self._catalog[name] = handler self.keys.update({name}) - handler.verbosity = self.verbosity + self._ordered_keys.append(name) + def configField(self, name): if name not in self.keys: @@ -114,7 +116,15 @@ def _complete(self, name, current_value): def _order(self, keys): """Gives implementations control over the order in which they process - installation parameters.""" + installation parameters. A ConfigManager should restrict itself to + the ConfigFields it knows about and ensure each field is only + consulted once.""" + ordered_keys = [] + known_fields = list(self.keys & set(keys)) # only handle keys we know + for f in self._ordered_keys: # use this order + if f in known_fields: # only fields the caller wants sorted + ordered_keys.append(f) + known_fields.remove(f) # avoid duplicates return keys def complete(self, collection): @@ -129,33 +139,28 @@ def complete(self, collection): contribution.setdefault(k, v) collection.update(contribution) -class ConfigField(ObjectWithOutput): - defaultOk = True - default = None - user_prompt = 'Please enter something' - - def __init__(self, defaultOk=None, default=None, user_prompt=None, verbosity=1): - super(ConfigField, self).__init__(verbosity=verbosity) - self.defaultOk = self.__class__.defaultOk if defaultOk is None else defaultOk - self.default = self.__class__.default if default is None else default - self.user_prompt = self.__class__.user_prompt if user_prompt is None else user_prompt - - def acceptable(self, value): - """High level sanity check for a specific value. This method is called - for an installation parameter with the value provided by the user, or - the default value, if the user didn't provide any value. There must be - a boolean response, if the installation can proceed with :var:value as - the setting for this ConfigField.""" - #self.print(f'This is {cls.__name__}.acceptable({value}) {cls.defaultOk}', 2) - if value is None and self.default is None or value == self.default: - return self.defaultOk - return True - - def ask_user(self, current): - """Prompt the user to provide a value for this installation - parameter.""" - user_prompt = self.user_prompt - if self.defaultOk is True: - user_prompt += ' (Just press ENTER, to use the current '\ - + f'value "{current}")' - return console.simple_dialog(user_prompt) + +# one could make a case for not deriving ConfigManagerCollection from +# ConfigManager because the collection serves a different purpose than the +# individual manager, but they are still quite similar +class ConfigManagerCollection(ConfigManager): + """ + Container class for ConfigManagers. + """ + def __init__(self, interactive=False, verbosity=1): + super(ConfigManagerCollection, self).__init__(interactive=interactive, verbosity=verbosity) + + def configManager(self, name): + return super(ConfigManagerCollection, self).configField(name) + + def complete(self, *args, **kwargs): + for manager in self._order(self.keys): + handler = self.configManager(manager) + handler.complete(*args, **kwargs) + + # these should never be called. we keep these implementations, just in case + def _remember(self, name, value): + raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') + + def _complete(self, name, value): + raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') diff --git a/askbot/deployment/base/deployablecomponent.py b/askbot/deployment/base/deployablecomponent.py new file mode 100644 index 0000000000..d5076ea7af --- /dev/null +++ b/askbot/deployment/base/deployablecomponent.py @@ -0,0 +1,68 @@ +from askbot.deployment.base import ObjectWithOutput +from .deployableobject import DeployableDir as Directory + +class DeployableComponent(ObjectWithOutput): + """These constitute sensible deployment chunks of Askbot. For instance, + one would never deploy just a settings.py, because Django projects need + additional files as well.""" + default_name = 'unset' + contents = dict() + + def __init__(self, name=None, contents=None): + super(DeployableComponent, self).__init__(verbosity=2) + name = name if name is not None else self.__class__.default_name + contents = contents if contents is not None else self.__class__.contents + + self.name = name + self.contents = contents + self._src_dir = None + self._dst_dir = None + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + def _grow_deployment_tree(self, component): + todo = list() + for name, deployable in component.items(): + if isinstance(deployable,dict): + branch = self._grow_deployment_tree(deployable) + todo.append( + Directory(name, None, *branch) + ) + else: + todo.append(deployable(name)) + return todo + + def _root_deployment_tree(self, tree): + root = Directory(self.name, None, *tree) + root.src_path = self.src_dir + root.dst_path = self.dst_dir + root.context = self.context + root.forced_overwrite = self.forced_overwrite + root.skip_silently = self.skip_silently + root.verbosity = self.verbosity + return root + + @property + def src_dir(self): + return self._src_dir + + @src_dir.setter + def src_dir(self, value): + self._src_dir = value + + @property + def dst_dir(self): + return self._dst_dir + + @dst_dir.setter + def dst_dir(self, value): + self._dst_dir = value + + def deploy(self): + """Recursively deploy all DeployItems we know about. This method is + meant to be overwritten by subclasses to provide more fine grained + configurations to individual branches and/or nodes.""" + tree = self._grow_deployment_tree(self.contents) + root = self._root_deployment_tree(tree) + root.deploy() diff --git a/askbot/deployment/base/deployableobject.py b/askbot/deployment/base/deployableobject.py new file mode 100644 index 0000000000..227c44b239 --- /dev/null +++ b/askbot/deployment/base/deployableobject.py @@ -0,0 +1,209 @@ +import os.path +import shutil + +from askbot.deployment.base.template_loader import DeploymentTemplate +from askbot.deployment.base.exceptions import AskbotDeploymentError +from askbot.deployment.base import ObjectWithOutput + +class DeployableObject(ObjectWithOutput): + """Base class for filesystem objects, i.e. files and directories, that can + be/are deployed using askbot-setup.""" + def __init__(self, name, src_path=None, dst_path=None): + super(DeployableObject, self).__init__(verbosity=2) + self.name = name + self._src_path = src_path + self._dst_path = dst_path + + @property + def src_path(self): + return self._src_path + + @src_path.setter + def src_path(self, value): + self._src_path = value + + @property + def dst_path(self): + return self._dst_path + + @dst_path.setter + def dst_path(self, value): + self._dst_path = value + + @property + def src(self): + return os.path.join(self.src_path, self.name) + + @property + def dst(self): + return os.path.join(self.dst_path, self.name) + + def _deploy_now(self): + """virtual protected""" + + def deploy(self): + """The main method of this class. DeployableComponents call this method + to have this object do the filesystem operations which deploys + whatever this class represents.""" + self.print(f'* {self.dst} from {self.src}') + try: + self._deploy_now() + except AskbotDeploymentError as e: + self.print(e) + +class DeployableFile(DeployableObject): + """This class collects all logic w.r.t. a single files. It has to be + subclassed and the subclasses must/should overwrite _deploy_now to call + one of the methods defined in this class. At the time of this writing, this + is either _render_with_jinja2 or _copy. + + The subclasses may then be used to deploy a single file. + """ + def __init__(self, name, src_path=None, dst_path=None): + super(DeployableFile, self).__init__(name, src_path, dst_path) + self.context = dict() + self.skip_silently = list() + self.forced_overwrite = list() + + # the different approaches for force and skip feel a little odd. + def __validate(self): + matches = [self.dst for c in self.forced_overwrite + if self.dst.endswith(f'{os.path.sep}{c}')] + exists = os.path.exists(self.dst) + force = len(matches) > 0 + skip = self.dst.split(os.path.sep)[-1] in self.skip_silently + return exists, force, skip + + def _render_with_jinja2(self, context): + exists, force, skip = self.__validate() + if exists: + raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') + template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended + template.tmpl_path = self.src + content = template.render(context) # do not put this in the context. If this statement fails, we do not want an empty self.dst to be created! + with open(self.dst, 'w+') as output_file: + output_file.write(content) + + def _copy(self): + exists, force, skip = self.__validate() + if exists: + if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. + return + elif force: + self.print(' ^^^ forced overwrite!') + else: + raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') + shutil.copy(self.src, self.dst) + + ##################### + ## from path_utils ## + ##################### + def _touch(self, times=None): + """implementation of unix ``touch`` in python""" + #http://stackoverflow.com/questions/1158076/implement-touch-using-python + try: + os.utime(self.dst, times) + except: + open(self.dst, 'a').close() + + +class DeployableDir(DeployableObject): + def __init__(self, name, parent=None, *content): + super(DeployableDir, self).__init__(name) + self.content = list(content) + if parent is not None: + self.dst_path = self.__clean_directory(parent) + + @DeployableObject.verbosity.setter + def verbosity(self, value): + self._verbosity = value + for child in self.content: + child.verbosity = value + + @DeployableObject.src_path.setter + def src_path(self, value): + value = self.__clean_directory(value) + self._src_path = value + for child in self.content: + child.src_path = value + + @DeployableObject.dst_path.setter + def dst_path(self, value): + value = self.__clean_directory(value) + self._dst_path = value + for child in self.content: + child.dst_path = self.dst + + @property + def context(self): + raise AttributeError(f'{self.__class__} does not store context information') + + @property + def skip_silently(self): + raise AttributeError(f'{self.__class__} does not store file skip information') + + @property + def forced_overwrite(self): + raise AttributeError(f'{self.__class__} does not store file overwrite information') + + # maybe deepcopy() in the following three setters. maybe there's an advantage in not deepcopy()ing here. time will tell. + @context.setter + def context(self, value): + for child in self.content: # in theory, we only want this for directories and rendered files. + child.context = value + + @skip_silently.setter + def skip_silently(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.skip_silently = value + + @forced_overwrite.setter + def forced_overwrite(self, value): + for child in self.content: # in practice, we only need this for directories and copied files. + child.forced_overwrite = value + + def _link_dir(self): + """Derived from __create_path()""" + if os.path.isdir(self.dst): + return + elif os.path.exists(self.dst): + raise AskbotDeploymentError('expect directory or a non-existing path') + else: + os.symlink(self.src, self.dst) + + ##################### + ## from path_utils ## + ##################### + def __create_path(self): + """equivalent to mkdir -p""" + if os.path.isdir(self.dst): + return + elif os.path.exists(self.dst): + raise AskbotDeploymentError('expect directory or a non-existing path') + else: + os.makedirs(self.dst) + + @staticmethod + def __clean_directory(directory): + """Returns normalized absolute path to the directory + regardless of whether it exists or not + or ``None`` - if the path is a file or if ``directory`` + parameter is ``None``""" + if directory is None: + return None + + directory = os.path.normpath(directory) + directory = os.path.abspath(directory) + + #if os.path.isfile(directory): + # print(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) + # return None + return directory + + def _deploy_now(self): + self.__create_path() + + def deploy(self): + self._deploy_now() # we don't try-except this, b/c input validation is supposed to identify problems before this code runs. If an exception is raised here, it's an issue that must be handled elsewhere. + for node in self.content: + node.deploy() diff --git a/askbot/deployment/base/exceptions.py b/askbot/deployment/base/exceptions.py new file mode 100644 index 0000000000..3654f10beb --- /dev/null +++ b/askbot/deployment/base/exceptions.py @@ -0,0 +1,23 @@ +class DirNameError(Exception): + """There is something about the chosen install dir we don't like.""" + +class RestrictionsError(DirNameError): + """The install dir name does not meet our requirements.""" + +class NameCollisionError(DirNameError): + """There is already a module with that name.""" + +class IsFileError(DirNameError): + """Cannot use that path.""" + +class CreateWriteError(DirNameError): + """Cannot write there.""" + +class NestedProjectsError(DirNameError): + """Cannot do a sensible deployment.""" + +class OverwriteError(DirNameError): + """This would overwrite things we don't want to overwrite.""" + +class AskbotDeploymentError(Exception): + """Use this when something goes wrong while deploying Askbot""" diff --git a/askbot/deployment/common/base.py b/askbot/deployment/base/objectwithoutput.py similarity index 79% rename from askbot/deployment/common/base.py rename to askbot/deployment/base/objectwithoutput.py index e1777cb104..c1e79e17e5 100644 --- a/askbot/deployment/common/base.py +++ b/askbot/deployment/base/objectwithoutput.py @@ -1,7 +1,3 @@ -class AskbotDeploymentError(Exception): - """Use this when something goes wrong while deploying Askbot""" - - class ObjectWithOutput(object): def __init__(self, verbosity=1, force=False): self._verbosity = verbosity diff --git a/askbot/deployment/template_loader.py b/askbot/deployment/base/template_loader.py similarity index 100% rename from askbot/deployment/template_loader.py rename to askbot/deployment/base/template_loader.py From cd6dca4e25d90653711885f38914738b1a937c3f Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 10 Oct 2019 22:48:03 +0200 Subject: [PATCH 43/45] Refactor installer [2/3] -- use base module * removed all classes and logic from the deployables and parameters modules that are provided through the base module * import from base module --- askbot/deployment/deployables/__init__.py | 3 +- askbot/deployment/deployables/components.py | 78 ++---- askbot/deployment/deployables/objects.py | 229 ++---------------- askbot/deployment/parameters/cache.py | 44 +--- .../deployment/parameters/configmanagers.py | 47 ++++ askbot/deployment/parameters/database.py | 91 ++----- askbot/deployment/parameters/filesystem.py | 57 +---- 7 files changed, 123 insertions(+), 426 deletions(-) create mode 100644 askbot/deployment/parameters/configmanagers.py diff --git a/askbot/deployment/deployables/__init__.py b/askbot/deployment/deployables/__init__.py index 3315b6e4c6..f84f532d89 100644 --- a/askbot/deployment/deployables/__init__.py +++ b/askbot/deployment/deployables/__init__.py @@ -6,10 +6,9 @@ did up to the error is undone. Yet, this code has no means to undo anything. """ -from askbot.deployment.common.base import AskbotDeploymentError from .objects import RenderedFile, CopiedFile, EmptyFile, Directory, LinkedDir from .components import AskbotApp, AskbotSite, ProjectRoot __all__ = ['RenderedFile', 'CopiedFile', 'EmptyFile', 'Directory', 'LinkedDir', - 'AskbotApp', 'AskbotSite', 'ProjectRoot', 'AskbotDeploymentError'] + 'AskbotApp', 'AskbotSite', 'ProjectRoot'] diff --git a/askbot/deployment/deployables/components.py b/askbot/deployment/deployables/components.py index d7ec4b49ba..65a5731236 100644 --- a/askbot/deployment/deployables/components.py +++ b/askbot/deployment/deployables/components.py @@ -1,71 +1,21 @@ -from askbot.deployment.deployables.objects import * -from askbot.deployment.common.base import ObjectWithOutput +import os -class DeployableComponent(ObjectWithOutput): - """These constitute sensible deployment chunks of Askbot. For instance, - one would never deploy just a settings.py, because Django projects need - additional files as well.""" - default_name = 'unset' - contents = dict() +from .objects import * +from askbot.deployment.base import DeployableComponent - def __init__(self, name=None, contents=None): - super(DeployableComponent, self).__init__(verbosity=2) - name = name if name is not None else self.__class__.default_name - contents = contents if contents is not None else self.__class__.contents +"""The classes defined herein are what we want to do when we talk about +deploying askbot. - self.name = name - self.contents = contents - self._src_dir = None - self._dst_dir = None - self.context = dict() - self.skip_silently = list() - self.forced_overwrite = list() +We write dictionaries, mostly named "contents" where we specify file/directory +names as keys and put the deployment object type as the value. The deployment +object type is basically how a particular filesystem object is to be created. +Nested dictionaries will result in a nested directory structure. - def _grow_deployment_tree(self, component): - todo = list() - for name, deployable in component.items(): - if isinstance(deployable,dict): - branch = self._grow_deployment_tree(deployable) - todo.append( - Directory(name, None, *branch) - ) - else: - todo.append(deployable(name)) - return todo - - def _root_deployment_tree(self, tree): - root = Directory(self.name, None, *tree) - root.src_path = self.src_dir - root.dst_path = self.dst_dir - root.context = self.context - root.forced_overwrite = self.forced_overwrite - root.skip_silently = self.skip_silently - root.verbosity = self.verbosity - return root - - @property - def src_dir(self): - return self._src_dir - - @src_dir.setter - def src_dir(self, value): - self._src_dir = value - - @property - def dst_dir(self): - return self._dst_dir - - @dst_dir.setter - def dst_dir(self, value): - self._dst_dir = value - - def deploy(self): - """Recursively deploy all DeployItems we know about. This method is - meant to be overwritten by subclasses to provide more fine grained - configurations to individual branches and/or nodes.""" - tree = self._grow_deployment_tree(self.contents) - root = self._root_deployment_tree(tree) - root.deploy() +Splitting the deployment into these three DeployableComponents is completely +arbitrary and seemed reasonable at the time. The intention was to keep +deployment details away from the main installer. However, the installer can +(and does) change the components before deploying them. +""" class AskbotSite(DeployableComponent): default_name = 'askbot_site' diff --git a/askbot/deployment/deployables/objects.py b/askbot/deployment/deployables/objects.py index 0f18c5b21a..3f7ad623ec 100644 --- a/askbot/deployment/deployables/objects.py +++ b/askbot/deployment/deployables/objects.py @@ -1,213 +1,10 @@ -import os.path -import shutil -from askbot.deployment.messages import print_message -from askbot.deployment.template_loader import DeploymentTemplate -from askbot.deployment.common.base import AskbotDeploymentError, ObjectWithOutput +from askbot.deployment.base import DeployableFile, DeployableDir -class DeployObject(ObjectWithOutput): - """Base class for filesystem objects, i.e. files and directories, that can - be/are deployed using askbot-setup.""" - def __init__(self, name, src_path=None, dst_path=None): - super(DeployObject, self).__init__(verbosity=2) - self.name = name - self._src_path = src_path - self._dst_path = dst_path - - @property - def src_path(self): - return self._src_path - - @src_path.setter - def src_path(self, value): - self._src_path = value - - @property - def dst_path(self): - return self._dst_path - - @dst_path.setter - def dst_path(self, value): - self._dst_path = value - - @property - def src(self): - return os.path.join(self.src_path, self.name) - - @property - def dst(self): - return os.path.join(self.dst_path, self.name) - - def _deploy_now(self): - """virtual protected""" - - def deploy(self): - """The main method of this class. DeployableComponents call this method - to have this object do the filesystem operations which deploys - whatever this class represents.""" - self.print(f'* {self.dst} from {self.src}') - try: - self._deploy_now() - except AskbotDeploymentError as e: - self.print(e) - -class DeployFile(DeployObject): - """This class collects all logic w.r.t. a single files. It has to be - subclassed and the subclasses must/should overwrite _deploy_now to call - one of the methods defined in this class. At the time of this writing, this - is either _render_with_jinja2 or _copy. - - The subclasses may then be used to deploy a single file. +class RenderedFile(DeployableFile): + """Render a file using Jinja2. + This class expects the source file to have a .jinja2-suffix and uses a + common render method. """ - def __init__(self, name, src_path=None, dst_path=None): - super(DeployFile, self).__init__(name, src_path, dst_path) - self.context = dict() - self.skip_silently = list() - self.forced_overwrite = list() - - # the different approaches for force and skip feel a little odd. - def __validate(self): - matches = [self.dst for c in self.forced_overwrite - if self.dst.endswith(f'{os.path.sep}{c}')] - exists = os.path.exists(self.dst) - force = len(matches) > 0 - skip = self.dst.split(os.path.sep)[-1] in self.skip_silently - return exists, force, skip - - def _render_with_jinja2(self, context): - exists, force, skip = self.__validate() - if exists: - raise AskbotDeploymentError(f' You already have a file "{self.dst}" please merge the contents.') - template = DeploymentTemplate('dummy.name') # we use this a little differently than originally intended - template.tmpl_path = self.src - content = template.render(context) # do not put this in the context. If this statement fails, we do not want an empty self.dst to be created! - with open(self.dst, 'w+') as output_file: - output_file.write(content) - - def _copy(self): - exists, force, skip = self.__validate() - if exists: - if skip: # this makes skip more important than force. This is good, because when in doubt, we rather leave the current installation the way it is. This is bad because we cannot explicitly force an overwrite. - return - elif force: - self.print(' ^^^ forced overwrite!') - else: - raise AskbotDeploymentError(f' You already have a file "{self.dst}", please add contents of {self.src}.') - shutil.copy(self.src, self.dst) - - ##################### - ## from path_utils ## - ##################### - def _touch(self, times=None): - """implementation of unix ``touch`` in python""" - #http://stackoverflow.com/questions/1158076/implement-touch-using-python - try: - os.utime(self.dst, times) - except: - open(self.dst, 'a').close() - - -class DeployDir(DeployObject): - def __init__(self, name, parent=None, *content): - super(DeployDir, self).__init__(name) - self.content = list(content) - if parent is not None: - self.dst_path = self.__clean_directory(parent) - - @DeployObject.verbosity.setter - def verbosity(self, value): - self._verbosity = value - for child in self.content: - child.verbosity = value - - @DeployObject.src_path.setter - def src_path(self, value): - value = self.__clean_directory(value) - self._src_path = value - for child in self.content: - child.src_path = value - - @DeployObject.dst_path.setter - def dst_path(self, value): - value = self.__clean_directory(value) - self._dst_path = value - for child in self.content: - child.dst_path = self.dst - - @property - def context(self): - raise AttributeError(f'{self.__class__} does not store context information') - - @property - def skip_silently(self): - raise AttributeError(f'{self.__class__} does not store file skip information') - - @property - def forced_overwrite(self): - raise AttributeError(f'{self.__class__} does not store file overwrite information') - - # maybe deepcopy() in the following three setters. maybe there's an advantage in not deepcopy()ing here. time will tell. - @context.setter - def context(self, value): - for child in self.content: # in theory, we only want this for directories and rendered files. - child.context = value - - @skip_silently.setter - def skip_silently(self, value): - for child in self.content: # in practice, we only need this for directories and copied files. - child.skip_silently = value - - @forced_overwrite.setter - def forced_overwrite(self, value): - for child in self.content: # in practice, we only need this for directories and copied files. - child.forced_overwrite = value - - def _link_dir(self): - """Derived from __create_path()""" - if os.path.isdir(self.dst): - return - elif os.path.exists(self.dst): - raise AskbotDeploymentError('expect directory or a non-existing path') - else: - os.symlink(self.src, self.dst) - - ##################### - ## from path_utils ## - ##################### - def __create_path(self): - """equivalent to mkdir -p""" - if os.path.isdir(self.dst): - return - elif os.path.exists(self.dst): - raise AskbotDeploymentError('expect directory or a non-existing path') - else: - os.makedirs(self.dst) - - @staticmethod - def __clean_directory(directory): - """Returns normalized absolute path to the directory - regardless of whether it exists or not - or ``None`` - if the path is a file or if ``directory`` - parameter is ``None``""" - if directory is None: - return None - - directory = os.path.normpath(directory) - directory = os.path.abspath(directory) - - #if os.path.isfile(directory): - # print(messages.CANT_INSTALL_INTO_FILE % {'path': directory}) - # return None - return directory - - def _deploy_now(self): - self.__create_path() - - def deploy(self): - self._deploy_now() # we don't try-except this, b/c input validation is supposed to identify problems before this code runs. If an exception is raised here, it's an issue that must be handled elsewhere. - for node in self.content: - node.deploy() - -class RenderedFile(DeployFile): def _deploy_now(self): self._render_with_jinja2(self.context) @@ -215,17 +12,23 @@ def _deploy_now(self): def src(self): return f'{super().src}.jinja2' -class CopiedFile(DeployFile): +class CopiedFile(DeployableFile): + """Copy a file from the source directory into the target directory.""" def _deploy_now(self): self._copy() -class EmptyFile(DeployFile): +class EmptyFile(DeployableFile): + """Create an empty file.""" def _deploy_now(self): self._touch() -class Directory(DeployDir): - pass +# Changing this file will not(!) alter the behaviour of deployable components, +# as it uses the base class DeployableDir. We only have the Directory for +# completeness sake. +class Directory(DeployableDir): + """Create an empty directory.""" -class LinkedDir(DeployDir): +class LinkedDir(DeployableDir): + """Create a symbolic link to an existing directory.""" def _deploy_now(self): self._link_dir() diff --git a/askbot/deployment/parameters/cache.py b/askbot/deployment/parameters/cache.py index 86d8ecd1e0..70735c525f 100644 --- a/askbot/deployment/parameters/cache.py +++ b/askbot/deployment/parameters/cache.py @@ -1,39 +1,5 @@ from askbot.utils import console -from askbot.deployment.parameters.base import ConfigField, ConfigManager - -class CacheConfigManager(ConfigManager): - """A config manager for validating setup parameters pertaining to - the cache Askbot will use.""" - def __init__(self, interactive=True, verbosity=1): - super(CacheConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) - db = ConfigField( - defaultOk = True, - user_prompt = 'Please enter the cache database name to use', - ) - password = ConfigField( - defaultOk = True, - user_prompt = 'Please enter the shared secret for accessing the cache', - ) - self.register('cache_engine', CacheEngine()) - self.register('cache_nodes', CacheNodes()) - self.register('cache_db', db) - self.register('cache_password', password) - - def _order(self, keys): - full_set = [ 'cache_engine', 'cache_nodes', 'cache_db', - 'cache_password' ] - return [ item for item in full_set if item in keys ] - - def _remember(self, name, value): - if name == 'cache_engine': - value = int(value) - super(CacheConfigManager, self)._remember(name, value) - if name == 'cache_engine': - if value == 3: - self._catalog['cache_nodes'].defaultOk = True - elif value == 2: - self._catalog['cache_db'].defaultOk = False - self._catalog['cache_password'].defaultOk = False +from askbot.deployment.base import ConfigField class CacheEngine(ConfigField): defaultOk = False @@ -76,3 +42,11 @@ class CacheNodes(ConfigField): def ask_user(self, current): value = super(CacheNodes, self).ask_user(current) return [ value ] + +class CacheDb(ConfigField): + defaultOk = True + user_prompt = 'Please enter the cache database name to use' + +class CachePass(ConfigField): + defaultOk=True + user_prompt='Please enter the shared secret for accessing the cache' diff --git a/askbot/deployment/parameters/configmanagers.py b/askbot/deployment/parameters/configmanagers.py new file mode 100644 index 0000000000..823169d459 --- /dev/null +++ b/askbot/deployment/parameters/configmanagers.py @@ -0,0 +1,47 @@ +from askbot.deployment.base import ConfigManager + +class CacheConfigManager(ConfigManager): + """A config manager for validating setup parameters pertaining to + the cache Askbot will use.""" + + def _remember(self, name, value): + if name == 'cache_engine': + value = int(value) + super(CacheConfigManager, self)._remember(name, value) + if name == 'cache_engine': + if value == 3: + self._catalog['cache_nodes'].defaultOk = True + elif value == 2: + self._catalog['cache_db'].defaultOk = False + self._catalog['cache_password'].defaultOk = False + +class DbConfigManager(ConfigManager): + """A config manager for validating setup parameters pertaining to + the database Askbot will use.""" + + def _remember(self, name, value): + if name == 'database_engine': + value = int(value) + super(DbConfigManager, self)._remember(name, value) + if name == 'database_engine': + self._catalog['database_name'].db_type = value + self._catalog['database_name'].set_user_prompt() + if value == 2: + self._catalog['database_user'].defaultOk = True + self._catalog['database_password'].defaultOk = True + + def _complete(self, name, current_value): + """ + Wrap the default _complete() and implement a special handling of + `database_engine`. While the user selects database engines by index, + i.e. 1,2,3 or 4, at the time of this writing, the installer and Askbot + use Django module names (I think that's what it is). Therefore we + perform a lookup after the user made their final choice and return + the name, rather than the index. + """ + ret = super(DbConfigManager, self)._complete(name, current_value) + if name == 'database_engine': + return [ e[1] for e in + self.configField(name).database_engines + if e[0] == ret ].pop() + return ret diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index c3abfddcaa..99ccd9851f 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -1,71 +1,27 @@ -from askbot.utils import console import os +from askbot.utils import console from askbot.deployment import path_utils +from askbot.deployment.parameters.base import ConfigField -from askbot.deployment.parameters.base import ConfigField, ConfigManager - -class DbConfigManager(ConfigManager): - """A config manager for validating setup parameters pertaining to - the database Askbot will use.""" - def __init__(self, interactive=True, verbosity=1): - super(DbConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) - engine = DbEngine() - name = DbName() - username = ConfigField( - defaultOk=False, - user_prompt='Please enter the username for accessing the database', - ) - password = ConfigField( - defaultOk=False, - user_prompt='Please enter the password for accessing the database', - ) - host = ConfigField( - defaultOk=True, - user_prompt='Please enter the database hostname', - ) - port = ConfigField( - defaultOk=True, - user_prompt='Please enter the database port', - ) +class DbUser(ConfigField): + defaultOk = False + user_prompt = 'Please enter the username for accessing the database' + + +class DbPass(ConfigField): + defaultOk = False + user_prompt = 'Please enter the password for accessing the database' + + +class DbHost(ConfigField): + defaultOk = True + user_prompt = 'Please enter the database hostname' + + +class DbPort(ConfigField): + defaultOk = True + user_prompt = 'Please enter the database port' - self.register('database_engine', engine) - self.register('database_name', name) - self.register('database_user', username) - self.register('database_password', password) - self.register('database_host', host) - self.register('database_port', port) - - def _order(self, keys): - full_set = [ 'database_engine', 'database_name', 'database_user', - 'database_password', 'database_host', 'database_port' ] - return [ item for item in full_set if item in keys ] - - def _remember(self, name, value): - if name == 'database_engine': - value = int(value) - super(DbConfigManager, self)._remember(name, value) - if name == 'database_engine': - self._catalog['database_name'].db_type = value - self._catalog['database_name'].set_user_prompt() - if value == 2: - self._catalog['database_user'].defaultOk = True - self._catalog['database_password'].defaultOk = True - - def _complete(self, name, current_value): - """ - Wrap the default _complete() to implement a special handling of - `database_engine`. While the user selects database engines by index, - i.e. 1,2,3 or 4, at the time of this writing, the installer and Askbot - use Django module names (I think that's what this is). Therefore we - perform a lookup after the user made their final choice and return - the name, rather than the index. - """ - ret = super(DbConfigManager, self)._complete(name, current_value) - if name == 'database_engine': - return [ e[1] for e in - self.configField(name).database_engines - if e[0] == ret ].pop() - return ret class DbEngine(ConfigField): defaultOk = False @@ -81,9 +37,8 @@ def acceptable(self, value): self.print(f'DbEngine.complete called with {value} of type {type(value)}', 2) try: return value in [e[0] for e in self.database_engines] - except: - pass - return False + finally: + return False def ask_user(self, current_value, depth=0): user_prompt = 'Please select database engine:\n' @@ -104,6 +59,7 @@ def ask_user(self, current_value, depth=0): return user_input + class DbName(ConfigField): defaultOk = False db_type = 1 @@ -132,4 +88,3 @@ def acceptable(self, value): else: return True return False - diff --git a/askbot/deployment/parameters/filesystem.py b/askbot/deployment/parameters/filesystem.py index 8a04d62754..b9db3e9bb1 100644 --- a/askbot/deployment/parameters/filesystem.py +++ b/askbot/deployment/parameters/filesystem.py @@ -1,56 +1,23 @@ -from askbot.utils import console -from askbot.deployment import messages -from askbot.deployment.parameters.base import ConfigField, ConfigManager -from askbot.deployment.path_utils import has_existing_django_project - from importlib.util import find_spec import os.path import re import tempfile +from askbot.utils import console +from askbot.deployment import messages +from askbot.deployment.parameters.base import ConfigField +from askbot.deployment.path_utils import has_existing_django_project +from askbot.deployment.base.exceptions import * DEBUG_VERBOSITY = 2 -class FilesystemConfigManager(ConfigManager): - """A config manager for validating setup parameters pertaining to - files and directories Askbot will use.""" - - def __init__(self, interactive=True, verbosity=1): - super(FilesystemConfigManager, self).__init__(interactive=interactive, verbosity=verbosity) - logfile = ConfigField( - defaultOk=True, - user_prompt="Please enter the name for Askbot's logfile.", - ) - self.register('dir_name', ProjectDirName()) - self.register('app_name', AppDirName()) - self.register('logfile_name', logfile) - - def _order(self, keys): - full_set = ['dir_name', 'app_name', 'logfile_name'] - return [item for item in full_set if item in keys] - -class DirNameError(Exception): - """There is something about the chosen install dir we don't like.""" - -class RestrictionsError(DirNameError): - pass - -class NameCollisionError(DirNameError): - pass - -class IsFileError(DirNameError): - pass - -class CreateWriteError(DirNameError): - pass - -class NestedProjectsError(DirNameError): - pass - -class OverwriteError(DirNameError): - pass +class LogfileName(ConfigField): + defaultOk = True + user_prompt = "Please enter the name for Askbot's logfile." class BaseDirName(ConfigField): - defaultOk = False + """Base class for directory parameters. + This extends ConfigField with a set of tests for directories and/or the + filesystem layout, to be used in ConfigField classes for directories.""" def _check_django_name_restrictions(self, directory): dir_name = os.path.basename(directory) @@ -94,6 +61,8 @@ def _check_forced_overwrite(self, directory): {'directory': directory}) class ProjectDirName(BaseDirName): + defaultOk = False + def acceptable(self, value): self.print(f'Got "{value}" of type "{type(value)}".', DEBUG_VERBOSITY) try: From d967fa55d3db98e904275c9a76dd91ede64f1a57 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Thu, 10 Oct 2019 22:51:21 +0200 Subject: [PATCH 44/45] Refactor installer [3/3] -- hardcode default * added comments to main installer class to describe what the important statements do * ALL the relations between input parameters (their names) and validation classes are defined in parameters/__init__.py; no more looking in different places. Is there validation for a specific input? This file holds the anser! Do you want to add validation for another paramter? Do it there! --- askbot/deployment/__init__.py | 24 ++++- askbot/deployment/parameters/__init__.py | 109 ++++++++++++++--------- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index f2911d8482..77283d799e 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -12,7 +12,7 @@ from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate from askbot.deployment.parameters import ConfigManagerCollection -from askbot.deployment.common.base import ObjectWithOutput +from askbot.deployment.base import ObjectWithOutput from askbot.deployment.deployables.components import DeployableComponent import askbot.deployment.deployables as deployable @@ -285,17 +285,30 @@ def __call__(self): # this is the main part of the original askbot_setup() nothing = DeployableComponent() nothing.deploy = lambda: None + # install into options['dir_name'] project = deployable.ProjectRoot(options['dir_name']) + + # select where to look for source files and templates project.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + + # set log dir an log file project.contents.update({ options['logdir_name']: {options['logfile_name']: deployable.EmptyFile} }) + # set the directory where settings.py etc. go, defaults to site = deployable.AskbotSite(options['app_name']) - site.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + + # install as a sub-directory to the intall directory site.dst_dir = options['dir_name'] + + # select where to look for source files and templates + site.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) + + # use user provided paramters to render files site.context.update(options) + # install container specifics, analogous to site uwsgi = deployable.AskbotApp() uwsgi.src_dir = os.path.join(self.ASKBOT_ROOT, self.SOURCE_DIR) uwsgi.dst_dir = options['dir_name'] @@ -304,13 +317,15 @@ def __call__(self): # this is the main part of the original askbot_setup() 'askbot_app': uwsgi.name, # defaults to askbot_app }) + # put the path to settings.py into manage.py project.context = {'settings_path': f'{site.name}.settings'} todo = [ project, site ] if options['create_project'] in ['no', 'none', 'false', '0', 'nothing']: - todo = [ nothing ] + todo = [ nothing ] # undocumented noop for the installer elif options['create_project'] == 'container-uwsgi': + # if we install into a container we additionally want these files project.contents.update({ 'cron': { 'crontab': deployable.RenderedFile, # askbot_site, askbot_app @@ -318,12 +333,15 @@ def __call__(self): # this is the main part of the original askbot_setup() }}) todo.append(uwsgi) + # maybe we could just use the noop nothing instead of this? if options['dry_run']: raise StopIteration + # install askbot for component in todo: component.deploy() + # the happily ever after section for successful deployments help_file = path_utils.get_path_to_help_file() self.print(messages.HOW_TO_DEPLOY_NEW % {'help_file': help_file}) diff --git a/askbot/deployment/parameters/__init__.py b/askbot/deployment/parameters/__init__.py index ed9ff3b549..f19e227c00 100644 --- a/askbot/deployment/parameters/__init__.py +++ b/askbot/deployment/parameters/__init__.py @@ -1,40 +1,69 @@ -from askbot.deployment.parameters.base import ConfigManager -from askbot.deployment.parameters.cache import CacheConfigManager -from askbot.deployment.parameters.database import DbConfigManager -from askbot.deployment.parameters.filesystem import FilesystemConfigManager - -# one could make a case for not deriving ConfigManagerCollection from -# ConfigManager because the collection serves a different purpose than the -# individual manager, but they are still quite similar -class ConfigManagerCollection(ConfigManager): - """ - This is the main config manager that will be used by the Askbot installer. - It is a hard coded ordered collection of all config managers the installer - shall use. - """ - def __init__(self, interactive=False, verbosity=0): - super(ConfigManagerCollection, self).__init__(interactive=interactive, verbosity=verbosity) - self.register('database', DbConfigManager(interactive=interactive, verbosity=verbosity)) - self.register('cache', CacheConfigManager(interactive=interactive, verbosity=verbosity)) - self.register('filesystem', FilesystemConfigManager(interactive=interactive, verbosity=verbosity)) - - def _order(self, keys): - full_set = ['filesystem', 'database', 'cache'] - return [item for item in full_set if item in keys] - - def configManager(self, name): - return super(ConfigManagerCollection, self).configField(name) - - def complete(self, *args, **kwargs): - for manager in self._order(self.keys): - handler = self.configManager(manager) - handler.complete(*args, **kwargs) - - # these should never be called. we keep these, just in case - def _remember(self, name, value): - raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') - - def _complete(self, name, value): - raise NotImplementedError(f'Not implemented in {self.__class__.__name__}.') - -__all__ = [ 'DbConfigManager', 'CacheConfigManager', 'FilesystemConfigManager', 'ConfigManagerCollection'] +from askbot.deployment.base import ConfigManagerCollection, ConfigManager +from .configmanagers import CacheConfigManager, DbConfigManager + +from .cache import * +from .database import * +from .filesystem import * + +""" +In this module we assemble the input validation capabilities for the Askbot +installer. + +The goal is to provide a single ConfigManagerCollection instance +(currently named askbotCollection), the installer will use to validate its +parameters. + +The idea is askbotCollection will be able to validate just the +parameters that are register()-ed in this module/file. First we create the +collection and managers. Then we register() all the parameters for which we +want validation. + +The validation implementations vary in complexity. Therefore, all +implementations are defined in derived classes in submodules, structured by +their topic, e.g. cache, database and filesystem. Here, at this level, we +import all classes and use register() as a mapping from parameter name, to +validation implementation. + +The parameter names must match the argpare argument destinations, i.e. the +`dest` argument to ArgumentParser.add_argument(), do be effective. + +THE ORDER IN WHICH VALIDATIONS ARE register()-ed WITH THEIR ConfigManagers +MATTERS! +""" + +# use these values while inizializing this module +interactive=False +verbosity=0 + +# the ConfigManagerCollection the installer will use +askbotCollection = ConfigManagerCollection(interactive=interactive, verbosity=verbosity) + +# the ConfigManagers the installer will use +cacheManager = CacheConfigManager(interactive=interactive, verbosity=verbosity) +databaseManager = DbConfigManager(interactive=interactive, verbosity=verbosity) +filesystemManager = ConfigManager(interactive=interactive, verbosity=verbosity) + +# register the ConfigManagers with the ConfigManagerCollection +askbotCollection.register('filesystem', filesystemManager) +askbotCollection.register('database', databaseManager) +askbotCollection.register('cache', cacheManager) + +# register parameters with config managers. THE ORDERING MATTERS! +cacheManager.register('cache_engine', CacheEngine()) +cacheManager.register('cache_nodes', CacheNodes()) +cacheManager.register('cache_db', CacheDb()) +cacheManager.register('cache_password', CachePass()) + +databaseManager.register('database_engine', DbEngine()) +databaseManager.register('database_name', DbName()) +databaseManager.register('database_user', DbUser()) +databaseManager.register('database_password', DbPass()) +databaseManager.register('database_host', DbHost()) +databaseManager.register('database_port', DbPort()) + +filesystemManager.register('dir_name', ProjectDirName()) +filesystemManager.register('app_name', AppDirName()) +filesystemManager.register('logfile_name', LogfileName()) + + +__all__ = ['askbotCollection', 'cacheManager', 'databaseManager', 'filesystemManager'] From 5cbf4e70d49ef05300c917a9aa270cd66d4d07e1 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sat, 16 Nov 2019 13:31:37 +0100 Subject: [PATCH 45/45] Lost patches and a bugfix * askbot/patches/__init__.py relied on some ancient method for obtaining the currently installed Django version => updated * Missing change: use the exported instances from askbot.deployment.parameters instead of creating new instances all the time * Missing change: add reset() to ConfigManger because for testing purposes we now need to reset ConfigManagers as we are not creating new instances for each test * Bugfix: ConfigManager._order(unsorted_list) would not return the result of its ordering, but the original unsorted_list, making this method effectively a noop. It now returns an ordered list as intended. --- askbot/deployment/__init__.py | 4 +- askbot/deployment/base/configmanager.py | 9 ++- .../deployment/parameters/configmanagers.py | 11 +++ askbot/deployment/parameters/database.py | 2 +- askbot/patches/__init__.py | 3 +- askbot/tests/test_installer.py | 74 +++++++++---------- 6 files changed, 59 insertions(+), 44 deletions(-) diff --git a/askbot/deployment/__init__.py b/askbot/deployment/__init__.py index 77283d799e..346d0c981e 100644 --- a/askbot/deployment/__init__.py +++ b/askbot/deployment/__init__.py @@ -11,7 +11,7 @@ from askbot.deployment import path_utils from askbot.utils.functions import generate_random_key from askbot.deployment.template_loader import DeploymentTemplate -from askbot.deployment.parameters import ConfigManagerCollection +from askbot.deployment.parameters import askbotCollection from askbot.deployment.base import ObjectWithOutput from askbot.deployment.deployables.components import DeployableComponent import askbot.deployment.deployables as deployable @@ -24,7 +24,7 @@ def __init__(self, interactive=True, verbosity=-128): super(AskbotSetup, self).__init__(verbosity=verbosity) self.parser = ArgumentParser(description="Setup a Django project and app for Askbot") self._todo = {} - self.configManagers = ConfigManagerCollection(interactive=interactive, verbosity=verbosity) + self.configManagers = askbotCollection self.database_engines = self.configManagers.configManager( 'database').configField( 'database_engine').database_engines diff --git a/askbot/deployment/base/configmanager.py b/askbot/deployment/base/configmanager.py index aa8834433f..8ae9b2097c 100644 --- a/askbot/deployment/base/configmanager.py +++ b/askbot/deployment/base/configmanager.py @@ -125,7 +125,7 @@ def _order(self, keys): if f in known_fields: # only fields the caller wants sorted ordered_keys.append(f) known_fields.remove(f) # avoid duplicates - return keys + return ordered_keys def complete(self, collection): """Main method of this :class:ConfigManager. @@ -139,6 +139,13 @@ def complete(self, collection): contribution.setdefault(k, v) collection.update(contribution) + def reset(self): + """ConfigManagers may keep a state. This method shall be used to reset + the state to whatever that means for the specific config manager. This + implementation merely flushes the memory about completed work.""" + self._managed_config = dict() + + # one could make a case for not deriving ConfigManagerCollection from # ConfigManager because the collection serves a different purpose than the diff --git a/askbot/deployment/parameters/configmanagers.py b/askbot/deployment/parameters/configmanagers.py index 823169d459..afdee7fc4b 100644 --- a/askbot/deployment/parameters/configmanagers.py +++ b/askbot/deployment/parameters/configmanagers.py @@ -15,10 +15,21 @@ def _remember(self, name, value): self._catalog['cache_db'].defaultOk = False self._catalog['cache_password'].defaultOk = False + def reset(self): + super(CacheConfigManager, self).reset() + self._catalog['cache_nodes'].defaultOk = False + self._catalog['cache_db'].defaultOk = True + self._catalog['cache_password'].defaultOk = True + class DbConfigManager(ConfigManager): """A config manager for validating setup parameters pertaining to the database Askbot will use.""" + def reset(self): + super(DbConfigManager, self).reset() + self._catalog['database_user'].defaultOk = False + self._catalog['database_password'].defaultOk = False + def _remember(self, name, value): if name == 'database_engine': value = int(value) diff --git a/askbot/deployment/parameters/database.py b/askbot/deployment/parameters/database.py index 99ccd9851f..a9753b3159 100644 --- a/askbot/deployment/parameters/database.py +++ b/askbot/deployment/parameters/database.py @@ -37,7 +37,7 @@ def acceptable(self, value): self.print(f'DbEngine.complete called with {value} of type {type(value)}', 2) try: return value in [e[0] for e in self.database_engines] - finally: + except AttributeError: return False def ask_user(self, current_value, depth=0): diff --git a/askbot/patches/__init__.py b/askbot/patches/__init__.py index daa9c5bcb0..70f377fffc 100644 --- a/askbot/patches/__init__.py +++ b/askbot/patches/__init__.py @@ -4,10 +4,9 @@ """ import django from askbot.patches import django_patches -from askbot.deployment import package_utils def patch_django(): - (major, minor, micro) = package_utils.get_django_version() + (major, minor, micro, _, __) = django.VERSION if major == 1 and minor > 4: # This shouldn't be required with django < 1.4.x diff --git a/askbot/tests/test_installer.py b/askbot/tests/test_installer.py index 6094a5662d..7932464549 100644 --- a/askbot/tests/test_installer.py +++ b/askbot/tests/test_installer.py @@ -29,7 +29,7 @@ def test_get_options(self): default_dict = vars(default_opts) def test_db_configmanager(self): - manager = DbConfigManager(interactive=False, verbosity=0) + manager = databaseManager new_empty = lambda:dict([(k,None) for k in manager.keys]) parameters = new_empty() # includes ALL database parameters @@ -54,29 +54,29 @@ class DatabaseEngineTest(AskbotTestCase): def setUp(self): self.installer = AskbotSetup() self.parser = self.installer.parser + self.manager = databaseManager + self.manager.reset() def test_database_engine(self): - manager = DbConfigManager(interactive=False, verbosity=0) - new_empty = lambda: dict([(k, None) for k in manager.keys]) - + new_empty = lambda: dict([(k, None) for k in self.manager.keys]) # DbConfigManager is supposed to test database_engine first # here: engine is NOT acceptable and name is NOT acceptable parameters = new_empty() # includes ALL database parameters try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as e: self.assertIn('database_engine', str(e)) # With a database_engine set, users must provide a database_name # here: engine is acceptable and name is NOT acceptable - engines = manager._catalog['database_engine'].database_engines + engines = self.manager._catalog['database_engine'].database_engines parameters = {'database_engine': None, 'database_name': None} caught_exceptions = 0 for db_type in [e[0] for e in engines]: parameters['database_engine'] = db_type try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: caught_exceptions += 1 self.assertIn('database_name', str(ve)) @@ -86,17 +86,17 @@ def test_database_engine(self): parameters = {'database_engine': None, 'database_name': 'acceptable_value'} e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve self.assertIn('database_engine', str(e)) # here: engine is acceptable and name is acceptable - acceptable_engine = manager._catalog['database_engine'].database_engines[0][0] + acceptable_engine = self.manager._catalog['database_engine'].database_engines[0][0] parameters = {'database_engine': acceptable_engine, 'database_name': 'acceptable_value'} e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve self.assertIsNone(e) @@ -104,27 +104,27 @@ def test_database_engine(self): # at the moment, the parameter parse does not have special code for # mysql and oracle, so we do not provide dedicated tests for them def test_database_postgres(self): - manager = DbConfigManager(interactive=False, verbosity=0) - new_empty = lambda: dict([(k, None) for k in manager.keys]) + new_empty = lambda: dict([(k, None) for k in self.manager.keys]) parameters = new_empty() parameters['database_engine'] = 1 - acceptable_answers = { - 'database_name': 'testDB', - 'database_user': 'askbot', - 'database_password': 'd34db33f', - } - expected_issues = acceptable_answers.keys() + ordered_acceptable_answers = ( + ('database_name', 'testDB'), + ('database_user', 'askbot'), + ('database_password', 'd34db33f'), + ) + + acceptable_answers = dict(ordered_acceptable_answers) + expected_issues = [item[0] for item in ordered_acceptable_answers] met_issues = set() for i in expected_issues: e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve matches = [issue for issue in expected_issues if issue in str(e)] self.assertEqual(len(matches), 1, str(e)) - self.assertIs(type(e), ValueError) issue = matches[0] cnt_old = len(met_issues) @@ -135,14 +135,13 @@ def test_database_postgres(self): self.assertEqual(len(expected_issues), len(met_issues)) e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve self.assertIsNone(e) def test_database_sqlite(self): - manager = DbConfigManager(interactive=False, verbosity=0) - new_empty = lambda: dict([(k, None) for k in manager.keys]) + new_empty = lambda: dict([(k, None) for k in self.manager.keys]) parameters = new_empty() parameters['database_engine'] = 2 @@ -154,7 +153,7 @@ def test_database_sqlite(self): for i in expected_issues: e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve matches = [issue for issue in expected_issues if issue in str(e)] @@ -170,7 +169,7 @@ def test_database_sqlite(self): self.assertEqual(len(expected_issues), len(met_issues)) e = None try: - manager.complete(parameters) + self.manager.complete(parameters) except ValueError as ve: e = ve self.assertIsNone(e) @@ -181,13 +180,13 @@ class CacheEngineTest(AskbotTestCase): def setUp(self): self.installer = AskbotSetup() self.parser = self.installer.parser + self.manager = cacheManager + self.manager.reset() - @staticmethod - def _setUpTest(): - manager = CacheConfigManager(interactive=False, verbosity=0) - engines = manager._catalog['cache_engine'].cache_engines - new_empty = lambda: dict([(k, None) for k in manager.keys]) - return manager, engines, new_empty + def _setUpTest(self): + engines = self.manager._catalog['cache_engine'].cache_engines + new_empty = lambda: dict([(k, None) for k in self.manager.keys]) + return self.manager, engines, new_empty @staticmethod def run_complete(manager, parameters): @@ -325,12 +324,12 @@ class FilesystemTests(AskbotTestCase): def setUp(self): self.installer = AskbotSetup() self.parser = self.installer.parser + self.manager = filesystemManager + self.manager.reset() - @staticmethod - def _setUpTest(): - manager = FilesystemConfigManager(interactive=False, verbosity=0) - new_empty = lambda: dict([(k, None) for k in manager.keys]) - return manager, new_empty + def _setUpTest(self): + new_empty = lambda: dict([(k, None) for k in self.manager.keys]) + return self.manager, new_empty @staticmethod def run_complete(manager, parameters): @@ -415,12 +414,11 @@ def __test_project_dir_interactive(self): """The console functions contain endless loops, which impedes testability. If we mock the console functions, then there is no merrit in testing interactively at all.""" - manager = FilesystemConfigManager(interactive=True, verbosity=1) parameters = {'dir_name': ''} #with patch('askbot.utils.console.simple_dialog', return_value='validDeployment'), patch('askbot.utils.console.choice_dialog', return_value='yes'): with patch('builtins.input', new=MockInput('martin', 'yes')): - e = self.run_complete(manager, parameters) + e = self.run_complete(self.manager, parameters) self.assertIsNone(e) class MainInstallerTests(AskbotTestCase):