diff --git a/curator/_version.py b/curator/_version.py index 3be392bd..50efedc0 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.1' +__version__ = '8.0.2' diff --git a/curator/actions/__init__.py b/curator/actions/__init__.py index 27764c29..03c638dd 100644 --- a/curator/actions/__init__.py +++ b/curator/actions/__init__.py @@ -13,3 +13,22 @@ from curator.actions.rollover import Rollover from curator.actions.shrink import Shrink from curator.actions.snapshot import Snapshot, DeleteSnapshots, Restore + +CLASS_MAP = { + 'alias' : Alias, + 'allocation' : Allocation, + 'close' : Close, + 'cluster_routing' : ClusterRouting, + 'create_index' : CreateIndex, + 'delete_indices' : DeleteIndices, + 'delete_snapshots' : DeleteSnapshots, + 'forcemerge' : ForceMerge, + 'index_settings' : IndexSettings, + 'open' : Open, + 'reindex' : Reindex, + 'replicas' : Replicas, + 'restore' : Restore, + 'rollover' : Rollover, + 'snapshot' : Snapshot, + 'shrink' : Shrink, +} diff --git a/curator/classdef.py b/curator/classdef.py new file mode 100644 index 00000000..6d4e6164 --- /dev/null +++ b/curator/classdef.py @@ -0,0 +1,227 @@ +"""Other Classes""" +import logging +from es_client.exceptions import ConfigurationError as ESclient_ConfigError +from es_client.helpers.utils import get_yaml +from curator import IndexList, SnapshotList +from curator.actions import CLASS_MAP +from curator.exceptions import ConfigurationError +from curator.config_utils import password_filter +from curator.helpers.testers import validate_actions + +# Let me tell you the story of the nearly wasted afternoon and the research that went into this +# seemingly simple work-around. Actually, no. It's even more wasted time writing that story here. +# Suffice to say that I couldn't use the CLASS_MAP with class objects to directly map them to class +# instances. The Wrapper class and the ActionDef.instantiate method do all of the work for me, +# allowing me to easily and cleanly pass *args and **kwargs to the individual action classes of +# CLASS_MAP. + +class Wrapper: + """Wrapper Class""" + def __init__(self, cls): + """Instantiate with passed Class (not instance or object) + + :param cls: A class (not an instance of the class) + """ + #: The class itself (not an instance of it), passed from ``cls`` + self.class_object = cls + #: An instance of :py:attr:`class_object` + self.class_instance = None + + def set_instance(self, *args, **kwargs): + """Set up :py:attr:`class_instance` from :py:attr:`class_object`""" + self.class_instance = self.class_object(*args, **kwargs) + + def get_instance(self, *args, **kwargs): + """Return the instance with ``*args`` and ``**kwargs`` + """ + self.set_instance(*args, **kwargs) + return self.class_instance + +class ActionsFile: + """Class to parse and verify entire actions file + + Individual actions are :py:class:`~.curator.classdef.ActionDef` objects + """ + def __init__(self, action_file): + self.logger = logging.getLogger(__name__) + #: The full, validated configuration from ``action_file``. + self.fullconfig = self.get_validated(action_file) + self.logger.debug('Action Configuration: %s', password_filter(self.fullconfig)) + #: A dict of all actions in the provided configuration. Each original key name is preserved + #: and the value is now an :py:class:`~.curator.classdef.ActionDef`, rather than a dict. + self.actions = None + self.set_actions(self.fullconfig['actions']) + + def get_validated(self, action_file): + """ + :param action_file: The path to a valid YAML action configuration file + :type action_file: str + + :returns: The result from passing ``action_file`` to + :py:func:`~.curator.helpers.testers.validate_actions` + """ + try: + return validate_actions(get_yaml(action_file)) + except (ESclient_ConfigError, UnboundLocalError) as err: + self.logger.critical('Configuration Error: %s', err) + raise ConfigurationError from err + + def parse_actions(self, all_actions): + """Parse the individual actions found in ``all_actions['actions']`` + + :param all_actions: All actions, each its own dictionary behind a numeric key. Making the + keys numeric guarantees that if they are sorted, they will always be executed in order. + + :type all_actions: dict + + :returns: + :rtype: list of :py:class:`~.curator.classdef.ActionDef` + """ + acts = {} + for idx in all_actions.keys(): + acts[idx] = ActionDef(all_actions[idx]) + return acts + + def set_actions(self, all_actions): + """Set the actions via :py:meth:`~.curator.classdef.ActionsFile.parse_actions` + + :param all_actions: All actions, each its own dictionary behind a numeric key. Making the + keys numeric guarantees that if they are sorted, they will always be executed in order. + :type all_actions: dict + :rtype: None + """ + self.actions = self.parse_actions(all_actions) + +# In this case, I just don't care that pylint thinks I'm overdoing it with attributes +# pylint: disable=too-many-instance-attributes +class ActionDef: + """Individual Action Definition Class + + Instances of this class represent an individual action from an action file. + """ + def __init__(self, action_dict): + #: The whole action dictionary + self.action_dict = action_dict + #: The action name + self.action = None + #: The action's class (Alias, Allocation, etc.) + self.action_cls = None + #: Only when action is alias will this be a :py:class:`~.curator.IndexList` + self.alias_adds = None + #: Only when action is alias will this be a :py:class:`~.curator.IndexList` + self.alias_removes = None + #: The list class, either :py:class:`~.curator.IndexList` or + #: :py:class:`~.curator.SnapshotList`. Default is :py:class:`~.curator.IndexList` + self.list_obj = Wrapper(IndexList) + #: The action ``description`` + self.description = None + #: The action ``options`` :py:class:`dict` + self.options = {} + #: The action ``filters`` :py:class:`list` + self.filters = None + #: The action option ``disable_action`` + self.disabled = None + #: The action option ``continue_if_exception`` + self.cif = None + #: The action option ``timeout_override`` + self.timeout_override = None + #: The action option ``ignore_empty_list`` + self.iel = None + #: The action option ``allow_ilm_indices`` + self.allow_ilm = None + self.set_root_attrs() + self.set_option_attrs() + self.log_the_options() + self.get_action_class() + + def instantiate(self, attribute, *args, **kwargs): + """ + Convert ``attribute`` from being a :py:class:`~.curator.classdef.Wrapper` of a Class to an + instantiated object of that Class. + + This is madness or genius. You decide. This entire method plus the + :py:class:`~.curator.classdef.Wrapper` class came about because I couldn't cleanly + instantiate a class variable into a class object. It works, and that's good enough for me. + + :param attribute: The `name` of an attribute that references a Wrapper class instance + :type attribute: str + """ + try: + wrapper = getattr(self, attribute) + except AttributeError as exc: + raise AttributeError(f'Bad Attribute: {attribute}. Exception: {exc}') from exc + setattr(self, attribute, self.get_obj_instance(wrapper, *args, **kwargs)) + + def get_obj_instance(self, wrapper, *args, **kwargs): + """Get the class instance wrapper identified by ``wrapper`` + Pass all other args and kwargs to the :py:meth:`~.curator.classdef.Wrapper.get_instance` + method. + + :returns: An instance of the class that :py:class:`~.curator.classdef.Wrapper` is wrapping + """ + if not isinstance(wrapper, Wrapper): + raise ConfigurationError( + f'{__name__} was passed wrapper which was of type {type(wrapper)}') + return wrapper.get_instance(*args, **kwargs) + + def set_alias_extras(self): + """Populate the :py:attr:`alias_adds` and :py:attr:`alias_removes` attributes""" + self.alias_adds = Wrapper(IndexList) + self.alias_removes = Wrapper(IndexList) + + def get_action_class(self): + """Get the action class from :py:const:`~.curator.actions.CLASS_MAP` + + Do extra setup when action is ``alias`` + + Set :py:attr:`list_obj` to :py:class:`~.curator.SnapshotList` when + :py:attr:`~.curator.classdef.ActionDef.action` is ``delete_snapshots`` or ``restore`` + """ + + self.action_cls = Wrapper(CLASS_MAP[self.action]) + if self.action == 'alias': + self.set_alias_extras() + if self.action in ['delete_snapshots', 'restore']: + self.list_obj = Wrapper(SnapshotList) + + def set_option_attrs(self): + """ + Iteratively get the keys and values from :py:attr:`~.curator.classdef.ActionDef.options` + and set the attributes + """ + attmap = { + 'disable_action': 'disabled', + 'continue_if_exception': 'cif', + 'ignore_empty_list': 'iel', + 'allow_ilm_indices': 'allow_ilm', + 'timeout_override' : 'timeout_override' + } + for key in self.action_dict['options']: + if key in attmap: + setattr(self, attmap[key], self.action_dict['options'][key]) + else: + self.options[key] = self.action_dict['options'][key] + + def set_root_attrs(self): + """ + Iteratively get the keys and values from + :py:attr:`~.curator.classdef.ActionDef.action_dict` and set the attributes + """ + for key, value in self.action_dict.items(): + # Gonna grab options in get_option_attrs() + if key == 'options': + continue + if value is not None: + setattr(self, key, value) + + def log_the_options(self): + """Log options at initialization time""" + logger = logging.getLogger('curator.cli.ActionDef') + msg = ( + f'For action {self.action}: disable_action={self.disabled}' + f'continue_if_exception={self.cif}, timeout_override={self.timeout_override}' + f'ignore_empty_list={self.iel}, allow_ilm_indices={self.allow_ilm}' + ) + logger.debug(msg) + if self.allow_ilm: + logger.warning('Permitting operation on indices with an ILM policy') diff --git a/curator/cli.py b/curator/cli.py index b8db8a06..3d067546 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -2,267 +2,344 @@ import sys import logging import click -from es_client.builder import ClientArgs, OtherArgs, Builder -from es_client.helpers.utils import get_yaml, check_config, prune_nones -from curator.actions import ( - Alias, Allocation, Close, ClusterRouting, CreateIndex, DeleteIndices, ForceMerge, - IndexSettings, Open, Reindex, Replicas, Rollover, Shrink, Snapshot, DeleteSnapshots, Restore -) -from curator.exceptions import ConfigurationError, ClientException -from curator.config_utils import check_logging_config, password_filter, set_logging +from es_client.builder import ClientArgs, OtherArgs +from es_client.helpers.utils import get_yaml, check_config, prune_nones, verify_url_schema +from curator.exceptions import ClientException +from curator.classdef import ActionsFile +from curator.config_utils import check_logging_config, set_logging from curator.defaults import settings from curator.exceptions import NoIndices, NoSnapshots -from curator.helpers.getters import get_write_index -from curator.helpers.testers import validate_actions -from curator.indexlist import IndexList -from curator.snapshotlist import SnapshotList +from curator.helpers.getters import get_client +from curator.helpers.testers import ilm_policy_check +from curator.cli_singletons.utils import get_width from curator._version import __version__ -CLASS_MAP = { - 'alias' : Alias, - 'allocation' : Allocation, - 'close' : Close, - 'cluster_routing' : ClusterRouting, - 'create_index' : CreateIndex, - 'delete_indices' : DeleteIndices, - 'delete_snapshots' : DeleteSnapshots, - 'forcemerge' : ForceMerge, - 'index_settings' : IndexSettings, - 'open' : Open, - 'reindex' : Reindex, - 'replicas' : Replicas, - 'restore' : Restore, - 'rollover' : Rollover, - 'snapshot' : Snapshot, - 'shrink' : Shrink, -} - -def process_action(client, config, **kwargs): - """ - Do the ``action`` in ``config``, using the associated options and ``kwargs``, if any. +def override_logging(config, loglevel, logfile, logformat): + """Get logging config and override from command-line options - :param config: ``action`` configuration data. + :param config: The configuration from file + :param loglevel: The log level + :param logfile: The log file to write + :param logformat: Which log format to use :type config: dict + :type loglevel: str + :type logfile: str + :type logformat: str + + :returns: Log configuration ready for validation + :rtype: dict + """ + # Check for log settings from config file + init_logcfg = check_logging_config(config) + + # Override anything with options from the command-line + if loglevel: + init_logcfg['loglevel'] = loglevel + if logfile: + init_logcfg['logfile'] = logfile + if logformat: + init_logcfg['logformat'] = logformat + return init_logcfg + +def cli_hostslist(hosts): + """ + :param hosts: One or more hosts. + :type hosts: str or list + + :returns: A list of hosts that came in from the command-line, or ``None`` + :rtype: list or ``None`` + """ + hostslist = [] + if hosts: + for host in list(hosts): + hostslist.append(verify_url_schema(host)) + else: + hostslist = None + return hostslist + +def ilm_action_skip(client, action_def): + """ + Skip rollover action if ``allow_ilm_indices`` is ``false``. For all other non-snapshot actions, + add the ``ilm`` filtertype to the :py:attr:`~.curator.ActionDef.filters` list. + + :param action_def: An action object + :type action_def: :py:class:`~.curator.classdef.ActionDef` + + :returns: ``True`` if ``action_def.action`` is ``rollover`` and the alias identified by + ``action_def.options['name']`` is associated with an ILM policy. This hacky work-around is + because the Rollover action does not use :py:class:`~.curator.IndexList` + :rtype: bool + """ + logger = logging.getLogger(__name__) + if not action_def.allow_ilm and action_def.action not in settings.snapshot_actions(): + if action_def.action == 'rollover': + if ilm_policy_check(client, action_def.options['name']): + logger.info('Alias %s is associated with ILM policy.', action_def.options['name']) + # logger.info('Skipping action %s because allow_ilm_indices is false.', idx) + return True + elif action_def.filters: + action_def.filters.append({'filtertype': 'ilm'}) + else: + action_def.filters = [{'filtertype': 'ilm'}] + return False + +def exception_handler(action_def, err): + """Do the grunt work with the exception + + :param action_def: An action object + :param err: The exception + + :type action_def: :py:class:`~.curator.classdef.ActionDef` + :type err: :py:exc:`Exception` + """ + logger = logging.getLogger(__name__) + if isinstance(err, (NoIndices, NoSnapshots)): + if action_def.iel: + logger.info( + 'Skipping action "%s" due to empty list: %s', action_def.action, type(err)) + else: + logger.error( + 'Unable to complete action "%s". No actionable items in list: %s', + action_def.action, type(err)) + sys.exit(1) + else: + logger.error( + 'Failed to complete action: %s. %s: %s', action_def.action, type(err), err) + if action_def.cif: + logger.info( + 'Continuing execution with next action because "continue_if_exception" ' + 'is set to True for action %s', action_def.action) + else: + sys.exit(1) + +def process_action(client, action_def, dry_run=False): + """ + Do the ``action`` in ``action_def.action``, using the associated options and any ``kwargs``. + + :param client: A client connection object + :param action_def: The ``action`` object + + :type client: :py:class:`~.elasticsearch.Elasticsearch` + :type action_def: :py:class:`~.curator.classdef.ActionDef` :rtype: None """ logger = logging.getLogger(__name__) - # Make some placeholder variables here for readability - logger.debug('Configuration dictionary: %s', config) - logger.debug('kwargs: %s', kwargs) - action = config['action'] - # This will always have some defaults now, so no need to do the if... - # # OLD WAY: opts = config['options'] if 'options' in config else {} - opts = config['options'] - logger.debug('opts: %s', opts) + logger.debug('Configuration dictionary: %s', action_def.action_dict) mykwargs = {} - action_class = CLASS_MAP[action] - # Add some settings to mykwargs... - if action == 'delete_indices': - mykwargs['master_timeout'] = ( - kwargs['master_timeout'] if 'master_timeout' in kwargs else 30) + if action_def.action == 'delete_indices': + mykwargs['master_timeout'] = 30 ### Update the defaults with whatever came with opts, minus any Nones - mykwargs.update(prune_nones(opts)) + mykwargs.update(prune_nones(action_def.options)) logger.debug('Action kwargs: %s', mykwargs) ### Set up the action ### - if action == 'alias': + logger.debug('Running "%s"', action_def.action.upper()) + if action_def.action == 'alias': # Special behavior for this action, as it has 2 index lists - logger.debug('Running "%s" action', action.upper()) - action_obj = action_class(**mykwargs) - removes = IndexList(client) - adds = IndexList(client) - if 'remove' in config: - logger.debug( - 'Removing indices from alias "%s"', opts['name']) - removes.iterate_filters(config['remove']) - action_obj.remove( - removes, warn_if_no_indices=opts['warn_if_no_indices']) - if 'add' in config: - logger.debug('Adding indices to alias "%s"', opts['name']) - adds.iterate_filters(config['add']) - action_obj.add(adds, warn_if_no_indices=opts['warn_if_no_indices']) - elif action in ['cluster_routing', 'create_index', 'rollover']: - action_obj = action_class(client, **mykwargs) - elif action in ['delete_snapshots', 'restore']: - logger.debug('Running "%s"', action) - slo = SnapshotList(client, repository=opts['repository']) - slo.iterate_filters(config) - # We don't need to send this value to the action - mykwargs.pop('repository') - action_obj = action_class(slo, **mykwargs) + action_def.instantiate('action_cls', **mykwargs) + action_def.instantiate('alias_adds', client) + action_def.instantiate('alias_removes', client) + if 'remove' in action_def.action_dict: + logger.debug('Removing indices from alias "%s"', action_def.options['name']) + action_def.alias_removes.iterate_filters(action_def.action_dict['remove']) + action_def.action_cls.remove( + action_def.alias_removes, + warn_if_no_indices=action_def.options['warn_if_no_indices']) + if 'add' in action_def.action_dict: + logger.debug('Adding indices to alias "%s"', action_def.options['name']) + action_def.alias_adds.iterate_filters(action_def.action_dict['add']) + action_def.action_cls.add( + action_def.alias_adds, warn_if_no_indices=action_def.options['warn_if_no_indices']) + elif action_def.action in ['cluster_routing', 'create_index', 'rollover']: + action_def.instantiate('action_cls', client, **mykwargs) else: - logger.debug('Running "%s"', action.upper()) - ilo = IndexList(client) - ilo.iterate_filters(config) - action_obj = action_class(ilo, **mykwargs) + if action_def.action in ['delete_snapshots', 'restore']: + mykwargs.pop('repository') # We don't need to send this value to the action + action_def.instantiate('list_obj', client, repository=action_def.options['repository']) + else: + action_def.instantiate('list_obj', client) + logger.debug('BLOCK1') + logger.debug('type is %s', type(action_def.list_obj)) + action_def.list_obj.iterate_filters({'filters': action_def.filters}) + logger.debug('BLOCK2 -- mykwargs = %s', mykwargs) + action_def.instantiate('action_cls', action_def.list_obj, **mykwargs) ### Do the action - if 'dry_run' in kwargs and kwargs['dry_run']: - action_obj.do_dry_run() + if dry_run: + action_def.action_cls.do_dry_run() else: logger.debug('Doing the action here.') - action_obj.do_action() + action_def.action_cls.do_action() -def run(config, action_file, dry_run=False): +def run(client_args, other_args, action_file, dry_run=False): """ Called by :py:func:`cli` to execute what was collected at the command-line + + :param client_args: The ClientArgs arguments object + :param other_args: The OtherArgs arguments object + :param action_file: The action configuration file + :param dry_run: Do not perform any changes + + :type client_args: :py:class:`~.es_client.ClientArgs` + :type other_args: :py:class:`~.es_client.OtherArgs` + :type action_file: str + :type dry_run: bool """ - # """Process yaml_file and return a valid client configuration""" - config_dict = get_yaml(config) - if config_dict is None: - click.echo('Empty configuration file provided. Using defaults') - config_dict = {} - elif not isinstance(config_dict, dict): - raise ConfigurationError('Configuration file not converted to dictionary. Check YAML configuration.') - set_logging(check_logging_config(config_dict)) - # set_logging({'loglevel':'DEBUG','blacklist':[]}) logger = logging.getLogger(__name__) - if not isinstance(config_dict, dict): - config_dict = {} - logger.warning( - 'Provided config file %s was unable to be properly read, or is empty. ' - 'Using empty dictionary (assuming defaults)', config) - logger.debug('config_dict = %s', config_dict) - client_args = ClientArgs() - other_args = OtherArgs() - if config: - raw_config = check_config(config_dict) - logger.debug('raw_config = %s', raw_config) - try: - client_args.update_settings(raw_config['client']) - # pylint: disable=broad-except - except Exception as exc: - click.echo(f'EXCEPTION = {exc}') - sys.exit(1) - other_args.update_settings(raw_config['other_settings']) - logger.debug('Client and logging configuration options validated.') - - ######################################### - ### Start working on the actions here ### - ######################################### logger.debug('action_file: %s', action_file) - action_config = get_yaml(action_file) - logger.debug('action_config: %s', password_filter(action_config)) - try: - action_dict = validate_actions(action_config) - except (ConfigurationError, UnboundLocalError) as err: - logger.critical('Configuration Error: %s', err) - click.echo(f'Critical configuration error encountered: {err}') - raise ConfigurationError from err - actions_config = action_dict['actions'] - logger.debug('Full list of actions: %s', password_filter(actions_config)) - action_keys = sorted(list(actions_config.keys())) - for idx in action_keys: - action = actions_config[idx]['action'] - action_disabled = actions_config[idx]['options'].pop('disable_action') - logger.debug('action_disabled = %s', action_disabled) - continue_if_exception = (actions_config[idx]['options'].pop('continue_if_exception')) - logger.debug('continue_if_exception = %s', continue_if_exception) - timeout_override = actions_config[idx]['options'].pop('timeout_override') - logger.debug('timeout_override = %s', timeout_override) - ignore_empty_list = actions_config[idx]['options'].pop('ignore_empty_list') - logger.debug('ignore_empty_list = %s', ignore_empty_list) - allow_ilm = actions_config[idx]['options'].pop('allow_ilm_indices') - logger.debug('allow_ilm_indices = %s', allow_ilm) - + all_actions = ActionsFile(action_file) + for idx in sorted(list(all_actions.actions.keys())): + action_def = all_actions.actions[idx] ### Skip to next action if 'disabled' - if action_disabled: + if action_def.disabled: logger.info( 'Action ID: %s: "%s" not performed because "disable_action" ' - 'is set to True', idx, action + 'is set to True', idx, action_def.action ) continue + logger.info('Preparing Action ID: %s, "%s"', idx, action_def.action) - logger.info('Preparing Action ID: %s, "%s"', idx, action) # Override the timeout, if specified, otherwise use the default. - if isinstance(timeout_override, int): - client_args.request_timeout = timeout_override - - # Set up action kwargs - kwargs = {} - kwargs['dry_run'] = dry_run + if action_def.timeout_override: + client_args.request_timeout = action_def.timeout_override # Create a client object for each action... logger.info('Creating client object and testing connection') - # Build a "final_config" that reflects CLI args overriding anything from a config_file - final_config = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } - builder = Builder(configdict=final_config) - try: - builder.connect() - except Exception as exc: - click.echo(f'Exception encountered: {exc}') - raise ClientException from exc + client = get_client(configdict={ + 'elasticsearch': { + 'client': prune_nones(client_args.asdict()), + 'other_settings': prune_nones(other_args.asdict()) + } + }) + except ClientException as exc: + # No matter where logging is set to go, make sure we dump these messages to the CLI + click.echo('Unable to establish client connection to Elasticsearch!') + click.echo(f'Exception: {exc}') + sys.exit(1) - client = builder.client ### Filter ILM indices unless expressly permitted - if allow_ilm: - logger.warning('allow_ilm_indices: true') - logger.warning('Permitting operation on indices with an ILM policy') - if not allow_ilm and action not in settings.snapshot_actions(): - if actions_config[idx]['action'] == 'rollover': - alias = actions_config[idx]['options']['name'] - write_index = get_write_index(client, alias) - try: - idx_settings = client.indices.get_settings(index=write_index) - if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: - logger.info('Alias %s is associated with ILM policy.', alias) - logger.info('Skipping action %s because allow_ilm_indices is false.', idx) - continue - except KeyError: - logger.debug('No ILM policies associated with %s', alias) - elif 'filters' in actions_config[idx]: - actions_config[idx]['filters'].append({'filtertype': 'ilm'}) - else: - actions_config[idx]['filters'] = [{'filtertype': 'ilm'}] + if ilm_action_skip(client, action_def): + continue ########################## ### Process the action ### ########################## + msg = f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' try: - logger.info( - 'Trying Action ID: %s, "%s": %s', idx, action, actions_config[idx]['description'] - ) - process_action(client, actions_config[idx], **kwargs) + logger.info(msg) + process_action(client, action_def, dry_run=dry_run) # pylint: disable=broad-except except Exception as err: - if isinstance(err, NoIndices) or isinstance(err, NoSnapshots): - if ignore_empty_list: - logger.info('Skipping action "%s" due to empty list: %s', action, type(err)) - else: - logger.error('Unable to complete action "%s". No actionable items in list: %s', action, type(err)) - sys.exit(1) - else: - logger.error('Failed to complete action: %s. %s: %s', action, type(err), err) - if continue_if_exception: - logger.info('Continuing execution with next action because "continue_if_exception" is set to True for action %s', action) - else: - sys.exit(1) - logger.info('Action ID: %s, "%s" completed.', idx, action) - logger.info('Job completed.') - -@click.command() -@click.option( - '--config', - help="Path to configuration file. Default: ~/.curator/curator.yml", - type=click.Path(exists=True), default=settings.config_file() -) + exception_handler(action_def, err) + logger.info('Action ID: %s, "%s" completed.', idx, action_def.action) + logger.info('All actions completed.') + +# pylint: disable=unused-argument, redefined-builtin +@click.command(context_settings=get_width()) +@click.option('--config', help='Path to configuration file.', type=click.Path(exists=True), default=settings.config_file()) +@click.option('--hosts', help='Elasticsearch URL to connect to', multiple=True) +@click.option('--cloud_id', help='Shorthand to connect to Elastic Cloud instance') +@click.option('--id', help='API Key "id" value', type=str) +@click.option('--api_key', help='API Key "api_key" value', type=str) +@click.option('--username', help='Username used to create "basic_auth" tuple') +@click.option('--password', help='Password used to create "basic_auth" tuple') +@click.option('--bearer_auth', type=str) +@click.option('--opaque_id', type=str) +@click.option('--request_timeout', help='Request timeout in seconds', type=float) +@click.option('--http_compress', help='Enable HTTP compression', is_flag=True, default=None) +@click.option('--verify_certs', help='Verify SSL/TLS certificate(s)', is_flag=True, default=None) +@click.option('--ca_certs', help='Path to CA certificate file or directory') +@click.option('--client_cert', help='Path to client certificate file') +@click.option('--client_key', help='Path to client certificate key') +@click.option('--ssl_assert_hostname', help='Hostname or IP address to verify on the node\'s certificate.', type=str) +@click.option('--ssl_assert_fingerprint', help='SHA-256 fingerprint of the node\'s certificate. If this value is given then root-of-trust verification isn\'t done and only the node\'s certificate fingerprint is verified.', type=str) +@click.option('--ssl_version', help='Minimum acceptable TLS/SSL version', type=str) +@click.option('--master-only', help='Only run if the single host provided is the elected master', is_flag=True, default=None) +@click.option('--skip_version_test', help='Do not check the host version', is_flag=True, default=None) @click.option('--dry-run', is_flag=True, help='Do not perform any changes.') +@click.option('--loglevel', help='Log level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])) +@click.option('--logfile', help='log file') +@click.option('--logformat', help='Log output format', type=click.Choice(['default', 'logstash', 'json', 'ecs'])) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(version=__version__) -def cli(config, dry_run, action_file): +@click.pass_context +def cli( + ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth, + opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, + ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, + dry_run, loglevel, logfile, logformat, action_file +): """ Curator for Elasticsearch indices. See http://elastic.co/guide/en/elasticsearch/client/curator/current """ - run(config, action_file, dry_run) + client_args = ClientArgs() + other_args = OtherArgs() + if config: + from_yaml = get_yaml(config) + raw_config = check_config(from_yaml) + client_args.update_settings(raw_config['client']) + other_args.update_settings(raw_config['other_settings']) + + set_logging(check_logging_config( + {'logging': override_logging(from_yaml, loglevel, logfile, logformat)})) + + hostslist = cli_hostslist(hosts) + + cli_client = prune_nones({ + 'hosts': hostslist, + 'cloud_id': cloud_id, + 'bearer_auth': bearer_auth, + 'opaque_id': opaque_id, + 'request_timeout': request_timeout, + 'http_compress': http_compress, + 'verify_certs': verify_certs, + 'ca_certs': ca_certs, + 'client_cert': client_cert, + 'client_key': client_key, + 'ssl_assert_hostname': ssl_assert_hostname, + 'ssl_assert_fingerprint': ssl_assert_fingerprint, + 'ssl_version': ssl_version + }) + + cli_other = prune_nones({ + 'master_only': master_only, + 'skip_version_test': skip_version_test, + 'username': username, + 'password': password, + 'api_key': { + 'id': id, + 'api_key': api_key + } + }) + # Remove `api_key` root key if `id` and `api_key` are both None + if id is None and api_key is None: + del cli_other['api_key'] + + # If hosts are in the config file, but cloud_id is specified at the command-line, + # we need to remove the hosts parameter as cloud_id and hosts are mutually exclusive + if cloud_id: + click.echo('cloud_id provided at CLI, superseding any other configured hosts') + client_args.hosts = None + cli_client.pop('hosts', None) + + # Likewise, if hosts are provided at the command-line, but cloud_id was in the config file, + # we need to remove the cloud_id parameter from the config file-based dictionary before merging + if hosts: + click.echo('hosts specified manually, superseding any other cloud_id or hosts') + client_args.hosts = None + client_args.cloud_id = None + cli_client.pop('cloud_id', None) + + # Update the objects if we have settings after pruning None values + if cli_client: + client_args.update_settings(cli_client) + if cli_other: + other_args.update_settings(cli_other) + run(client_args, other_args, action_file, dry_run) diff --git a/curator/helpers/getters.py b/curator/helpers/getters.py index c00d93a2..e7b776f3 100644 --- a/curator/helpers/getters.py +++ b/curator/helpers/getters.py @@ -2,7 +2,9 @@ # :pylint disable= import logging from elasticsearch8 import exceptions as es8exc -from curator.exceptions import CuratorException, FailedExecution, MissingArgument +from es_client.defaults import VERSION_MAX, VERSION_MIN +from es_client.builder import Builder +from curator.exceptions import ClientException, CuratorException, FailedExecution, MissingArgument def byte_size(num, suffix='B'): """ @@ -22,6 +24,46 @@ def byte_size(num, suffix='B'): num /= 1024.0 return f'{num:.1f}Y{suffix}' +def get_client( + configdict=None, configfile=None, autoconnect=False, version_min=VERSION_MIN, + version_max=VERSION_MAX): + """Get an Elasticsearch Client using :py:class:`es_client.Builder` + + Build a client out of settings from `configfile` or `configdict` + If neither `configfile` nor `configdict` is provided, empty defaults will be used. + If both are provided, `configdict` will be used, and `configfile` ignored. + + :param configdict: A configuration dictionary + :param configfile: A configuration file + :param autoconnect: Connect to client automatically + :param verion_min: Minimum acceptable version of Elasticsearch (major, minor, patch) + :param verion_max: Maximum acceptable version of Elasticsearch (major, minor, patch) + + :type configdict: dict + :type configfile: str + :type autoconnect: bool + :type version_min: tuple + :type version_max: tuple + + :returns: A client connection object + :rtype: :py:class:`~.elasticsearch.Elasticsearch` + """ + logger = logging.getLogger(__name__) + logger.info('Creating client object and testing connection') + + builder = Builder( + configdict=configdict, configfile=configfile, autoconnect=autoconnect, + version_min=version_min, version_max=version_max + ) + + try: + builder.connect() + except Exception as exc: + logger.critical('Exception encountered: %s', exc) + raise ClientException from exc + + return builder.client + def get_indices(client): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` diff --git a/curator/helpers/testers.py b/curator/helpers/testers.py index 40ba6fef..809e6e0e 100644 --- a/curator/helpers/testers.py +++ b/curator/helpers/testers.py @@ -4,13 +4,38 @@ from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import NotFoundError from es_client.helpers.utils import prune_nones -from curator.helpers.getters import get_repository +from curator.helpers.getters import get_repository, get_write_index from curator.exceptions import ConfigurationError, MissingArgument, RepositoryException from curator.defaults.settings import index_filtertypes, snapshot_actions, snapshot_filtertypes from curator.validators import SchemaCheck, actions, options from curator.validators.filter_functions import validfilters from curator.helpers.utils import report_failure +def ilm_policy_check(client, alias): + """Test if alias is associated with an ILM policy + + Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` + + :param client: A client connection object + :param alias: The alias name + + :type client: :py:class:`~.elasticsearch.Elasticsearch` + :type alias: str + :rtype: bool + """ + logger = logging.getLogger(__name__) + # alias = action_obj.options['name'] + write_index = get_write_index(client, alias) + try: + idx_settings = client.indices.get_settings(index=write_index) + if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: + # logger.info('Alias %s is associated with ILM policy.', alias) + # logger.info('Skipping action %s because allow_ilm_indices is false.', idx) + return True + except KeyError: + logger.debug('No ILM policies associated with %s', alias) + return False + def repository_exists(client, repository=None): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` diff --git a/curator/repomgrcli.py b/curator/repomgrcli.py index c573bfc5..1d7f589b 100644 --- a/curator/repomgrcli.py +++ b/curator/repomgrcli.py @@ -317,7 +317,7 @@ def source( @click.option('--ssl_version', help='Minimum acceptable TLS/SSL version', type=str) @click.option('--master-only', help='Only run if the single host provided is the elected master', is_flag=True, default=None) @click.option('--skip_version_test', help='Do not check the host version', is_flag=True, default=None) -@click.option('--dry-run', is_flag=True, help='Do not perform any changes.') +@click.option('--dry-run', is_flag=True, help='Do not perform any changes. NON-FUNCTIONAL PLACEHOLDER! DO NOT USE!') @click.option('--loglevel', help='Log level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])) @click.option('--logfile', help='log file') @click.option('--logformat', help='Log output format', type=click.Choice(['default', 'logstash', 'json', 'ecs'])) diff --git a/docker_test/scripts/create.sh b/docker_test/scripts/create.sh index 3bd8d1a3..fd1a54cc 100755 --- a/docker_test/scripts/create.sh +++ b/docker_test/scripts/create.sh @@ -128,6 +128,12 @@ echo for IP in $IPLIST; do echo "REMOTE_ES_SERVER=\"http://$IP:${REMOTE_PORT}\"" + if [ "$AUTO_EXPORT" == "y" ]; then + # This puts our curatortestenv file where it can be purged easily by destroy.sh + cd $SCRIPTPATH + cd .. + echo "export REMOTE_ES_SERVER=http://$IP:$REMOTE_PORT" > curatortestenv + fi done echo diff --git a/docker_test/scripts/destroy.sh b/docker_test/scripts/destroy.sh index 88aac3cd..a5b2bee0 100755 --- a/docker_test/scripts/destroy.sh +++ b/docker_test/scripts/destroy.sh @@ -26,6 +26,7 @@ UPONE=$(pwd | awk -F\/ '{print $NF}') if [[ "$UPONE" = "docker_test" ]]; then rm -rf $(pwd)/repo/* + rm -rf $(pwd)/curatortestenv else echo "WARNING: Unable to automatically empty bind mounted repo path." echo "Please manually empty the contents of the repo directory!" diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 5aae576d..e38f83e5 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,22 @@ Changelog ========= +8.0.2 (15 February 2023) +------------------------ + +**Changes** + + * Added the same CLI flags that the singletons offers. This gives much more flexibility with + regards to passing configuration settings as command-line options, particularly for Docker. + * Re-created the ``get_client`` function. It now resides in ``curator.helpers.getters`` and will + eventually see use in the Reindex class for remote connections. + * Created a new set of classes to import, validate the schema, and split individual actions into + their own sub-object instances. This is primarily to make ``curator/cli.py`` read much more + cleanly. No new functionality here, but fewer conditional branches, and hopefully more readable + code. + * Updated the documentation to show these changes, both the API and the Elastic.co usage docs. + + 8.0.1 (10 February 2023) ------------------------ diff --git a/docs/api.rst b/docs/api.rst index 133c497f..7b81368e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -9,6 +9,7 @@ API and Code Reference actions indexlist snapshotlist + classdef helpers validators defaults diff --git a/docs/asciidoc/command-line.asciidoc b/docs/asciidoc/command-line.asciidoc index ed73b0d4..ff0431a4 100644 --- a/docs/asciidoc/command-line.asciidoc +++ b/docs/asciidoc/command-line.asciidoc @@ -11,7 +11,13 @@ [[command-line]] == Command Line Interface -The command-line arguments are as follows: +Most common client configuration settings are now available at the command-line. + +IMPORTANT: While both the configuration file and the command-line arguments can be used together, + it is important to note that command-line options will override file-based configuration of the + same setting. + +The most basic command-line arguments are as follows: [source,sh] ------- @@ -33,7 +39,7 @@ specified. `ACTION_FILE.YML` is a YAML <>. -Command-line help is never far away: +For other client configuration options, command-line help is never far away: [source,sh] ------- @@ -52,10 +58,35 @@ Usage: curator [OPTIONS] ACTION_FILE See http://elastic.co/guide/en/elasticsearch/client/curator/current Options: - --config PATH Path to configuration file. Default: ~/.curator/curator.yml - --dry-run Do not perform any changes. - --version Show the version and exit. - --help Show this message and exit. + --config PATH Path to configuration file. + --hosts TEXT Elasticsearch URL to connect to + --cloud_id TEXT Shorthand to connect to Elastic Cloud instance + --id TEXT API Key "id" value + --api_key TEXT API Key "api_key" value + --username TEXT Username used to create "basic_auth" tuple + --password TEXT Password used to create "basic_auth" tuple + --bearer_auth TEXT + --opaque_id TEXT + --request_timeout FLOAT Request timeout in seconds + --http_compress Enable HTTP compression + --verify_certs Verify SSL/TLS certificate(s) + --ca_certs TEXT Path to CA certificate file or directory + --client_cert TEXT Path to client certificate file + --client_key TEXT Path to client certificate key + --ssl_assert_hostname TEXT Hostname or IP address to verify on the node's certificate. + --ssl_assert_fingerprint TEXT SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust + verification isn't done and only the node's certificate fingerprint is verified. + --ssl_version TEXT Minimum acceptable TLS/SSL version + --master-only Only run if the single host provided is the elected master + --skip_version_test Do not check the host version + --dry-run Do not perform any changes. + --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level + --logfile TEXT log file + --logformat [default|logstash|json|ecs] + Log output format + --version Show the version and exit. + --help Show this message and exit. ------- You can use <> in your configuration files. @@ -96,43 +127,61 @@ file, though it does support using the client configuration file if you want. As an important bonus, the command-line options allow you to override the settings in the `curator.yml` file! +IMPORTANT: While both the configuration file and the command-line arguments can be used together, + it is important to note that command-line options will override file-based configuration of the + same setting. + [source,sh] --------- $ curator_cli --help Usage: curator_cli [OPTIONS] COMMAND [ARGS]... Options: - --config PATH Path to configuration file. Default: - ~/.curator/curator.yml - --host TEXT Elasticsearch host. - --url_prefix TEXT Elasticsearch http url prefix. - --port TEXT Elasticsearch port. - --use_ssl Connect to Elasticsearch through SSL. - --certificate TEXT Path to certificate to use for SSL validation. - --client-cert TEXT Path to file containing SSL certificate for client auth. - --client-key TEXT Path to file containing SSL key for client auth. - --ssl-no-validate Do not validate SSL certificate - --http_auth TEXT Use Basic Authentication ex: user:pass - --timeout INTEGER Connection timeout in seconds. - --master-only Only operate on elected master node. - --dry-run Do not perform any changes. - --loglevel TEXT Log level - --logfile TEXT log file - --logformat TEXT Log output format [default|logstash|json|ecs]. - --version Show the version and exit. - --help Show this message and exit. + --config PATH Path to configuration file. + --hosts TEXT Elasticsearch URL to connect to + --cloud_id TEXT Shorthand to connect to Elastic Cloud instance + --id TEXT API Key "id" value + --api_key TEXT API Key "api_key" value + --username TEXT Username used to create "basic_auth" tuple + --password TEXT Password used to create "basic_auth" tuple + --bearer_auth TEXT + --opaque_id TEXT + --request_timeout FLOAT Request timeout in seconds + --http_compress Enable HTTP compression + --verify_certs Verify SSL/TLS certificate(s) + --ca_certs TEXT Path to CA certificate file or directory + --client_cert TEXT Path to client certificate file + --client_key TEXT Path to client certificate key + --ssl_assert_hostname TEXT Hostname or IP address to verify on the node's certificate. + --ssl_assert_fingerprint TEXT SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust + verification isn't done and only the node's certificate fingerprint is verified. + --ssl_version TEXT Minimum acceptable TLS/SSL version + --master-only Only run if the single host provided is the elected master + --skip_version_test Do not check the host version + --dry-run Do not perform any changes. + --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level + --logfile TEXT log file + --logformat [default|logstash|json|ecs] + Log output format + --version Show the version and exit. + --help Show this message and exit. Commands: + alias Add/Remove Indices to/from Alias allocation Shard Routing Allocation - close Close indices - delete-indices Delete indices - delete-snapshots Delete snapshots - forcemerge forceMerge index/shard segments - open Open indices - replicas Change replica count - show-indices Show indices - show-snapshots Show snapshots - snapshot Snapshot indices + close Close Indices + delete-indices Delete Indices + delete-snapshots Delete Snapshots + forcemerge forceMerge Indices (reduce segment count) + open Open Indices + replicas Change Replica Count + restore Restore Indices + rollover Rollover Index associated with Alias + show-indices Show Indices + show-snapshots Show Snapshots + shrink Shrink Indices to --number_of_shards + snapshot Snapshot Indices --------- The option flags for the given commands match those used for the same diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index e485578b..44127d2a 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,4 +1,4 @@ -:curator_version: 8.0.1 +:curator_version: 8.0.2 :curator_major: 8 :curator_doc_tree: 8.0 :es_py_version: 8.6.1 diff --git a/docs/classdef.rst b/docs/classdef.rst new file mode 100644 index 00000000..87b9e901 --- /dev/null +++ b/docs/classdef.rst @@ -0,0 +1,34 @@ +.. _classdef: + +Other Class Definitions +####################### + +.. _wrapper: + +Wrapper +======= + +.. autoclass:: curator.classdef.Wrapper + :members: + :undoc-members: + :show-inheritance: + +.. _actionfile: + +ActionsFile +=========== + +.. autoclass:: curator.classdef.ActionsFile + :members: + :undoc-members: + :show-inheritance: + +.. _actiondef: + +ActionDef +========= + +.. autoclass:: curator.classdef.ActionDef + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 31ce0bd8..34bb355a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,6 +72,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3.11', None), + 'es_client': ('https://es_client.readthedocs.io/en/v8.6.1.post1', None), 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.6.1', None), 'voluptuous': ('http://alecthomas.github.io/voluptuous/docs/_build/html', None), 'click': ('https://click.palletsprojects.com/en/8.1.x', None), diff --git a/docs/helpers.rst b/docs/helpers.rst index 76de844b..4cd48b72 100644 --- a/docs/helpers.rst +++ b/docs/helpers.rst @@ -50,6 +50,8 @@ Getters .. autofunction:: byte_size +.. autofunction:: get_client + .. autofunction:: get_indices .. autofunction:: get_repository @@ -77,6 +79,8 @@ Testers .. py:module:: curator.helpers.testers +.. autofunction:: ilm_policy_check + .. autofunction:: repository_exists .. autofunction:: rollable_alias @@ -92,6 +96,7 @@ Testers .. autofunction:: verify_repository .. autofunction:: verify_snapshot_list + .. _helpers_utils: Utils diff --git a/docs/other_modules.rst b/docs/other_modules.rst index 19b16150..8e95cc7f 100644 --- a/docs/other_modules.rst +++ b/docs/other_modules.rst @@ -10,19 +10,75 @@ Other Modules .. autofunction:: process_action +.. autofunction:: override_logging + +.. autofunction:: cli_hostslist + +.. autofunction:: ilm_action_skip + +.. autofunction:: exception_handler + .. autofunction:: run -.. py:function:: cli(config, dry_run, action_file) +.. py:function:: cli(ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, dry_run, loglevel, logfile, logformat, action_file) This is the :py:class:`click.Command` that initiates everything and connects the command-line to the rest of Curator. - :param config: Path to configuration file. Default: ~/.curator/curator.yml + :param ctx: The Click Context + :param config: Path to configuration file. + :param hosts: Elasticsearch URL to connect to + :param cloud_id: Shorthand to connect to Elastic Cloud instance + :param id: API Key "id" value + :param api_key: API Key "api_key" value + :param username: Username used to create "basic_auth" tuple + :param password: Password used to create "basic_auth" tuple + :param bearer_auth: Bearer Auth Token + :param opaque_id: Opaque ID string + :param request_timeout: Request timeout in seconds + :param http_compress: Enable HTTP compression + :param verify_certs: Verify SSL/TLS certificate(s) + :param ca_certs: Path to CA certificate file or directory + :param client_cert: Path to client certificate file + :param client_key: Path to client certificate key + :param ssl_assert_hostname: Hostname or IP address to verify on the node's certificate. + :param ssl_assert_fingerprint: SHA-256 fingerprint of the node's certificate. If this value is + given then root-of-trust verification isn't done and only the node's certificate + fingerprint is verified. + :param ssl_version: Minimum acceptable TLS/SSL version + :param master_only: Only run if the single host provided is the elected master + :param skip_version_test: Do not check the host version :param dry_run: Do not perform any changes. + :param loglevel: Log level + :param logfile: Path to log file + :param logformat: Log output format :param action_file: Path to action file + :type ctx: :py:class:`~.click.Context` :type config: str + :type hosts: list + :type cloud_id: str + :type id: str + :type api_key: str + :type username: str + :type password: str + :type bearer_auth: str + :type opaque_id: str + :type request_timeout: int + :type http_compress: bool + :type verify_certs: bool + :type ca_certs: str + :type client_cert: str + :type client_key: str + :type ssl_assert_hostname: str + :type ssl_assert_fingerprint: str + :type ssl_version: str + :type master_only: bool + :type skip_version_test: bool :type dry_run: bool + :type loglevel: str + :type logfile: str + :type logformat: str :type action_file: str ``curator.config_utils`` @@ -267,7 +323,7 @@ This inherits from :py:class:`logging.Formatter`, so some of what you see docume :param ssl_version: Minimum acceptable TLS/SSL version :param master_only: Only run if the single host provided is the elected master :param skip_version_test: Do not check the host version - :param dry_run: Do not perform any changes. + :param dry_run: Do not perform any changes. NON-FUNCTIONAL PLACEHOLDER! DO NOT USE! :param loglevel: Log level :param logfile: Path to log file :param logformat: Log output format diff --git a/docs/usage.rst b/docs/usage.rst index e5a6be4a..7cc8c080 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -21,7 +21,7 @@ Command-Line Usage ================== The documentation for this is on -`Elastic's Website `_. +`Elastic's Website `_. Example API Usage ================= diff --git a/pyproject.toml b/pyproject.toml index 8c653ea7..8d1c2a8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,29 @@ exclude = [ "tests", ] +[tool.hatch.envs.test] +dependencies = [ + "coverage[toml]", + "mock", + "requests", + "pytest >=7.2.1", + "pytest-cov", +] + +[tool.hatch.envs.test.scripts] +step0 = "$(docker_test/scripts/destroy.sh 2&>1 /dev/null)" +step1 = "step0 ; echo 'Starting test environment in Docker...' ; $(AUTO_EXPORT=y docker_test/scripts/create.sh 8.6.1 2&>1 /dev/null)" +step2 = "step1 ; source docker_test/curatortestenv; echo 'Running tests:'" +step3 = "step2 ; pytest ; EXITCODE=$?" +step4 = "step3 ; echo 'Tests complete! Destroying Docker test environment...' " +full = "step4 ; $(docker_test/scripts/destroy.sh 2&>1 /dev/null ) ; exit $EXITCODE" +run-coverage = "pytest --cov-config=pyproject.toml --cov=curator --cov=tests" +run = "run-coverage --no-cov" + +[[tool.hatch.envs.test.matrix]] +python = ["3.9", "3.10", "3.11"] +version = ["8.0.2"] + [tool.pytest.ini_options] pythonpath = [".", "curator"] minversion = "7.2" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 320b824a..a333d987 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -220,3 +220,13 @@ def invoke_runner(self, dry_run=False): ) return self.result = self.runner.invoke(cli, self.runner_args) + + def invoke_runner_alt(self, **kwargs): + myargs = [] + if kwargs: + for key, value in kwargs.items(): + myargs.append(f'--{key}') + myargs.append(value) + myargs.append(self.args['actionfile']) + self.result = self.runner.invoke(cli, myargs) + diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 70600988..d2168f31 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -15,6 +15,17 @@ def test_bad_client_config(self): self.write_config(self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices')) self.invoke_runner() assert 1 == self.result.exit_code + def test_cli_client_config(self): + self.create_indices(10) + self.write_config(self.args['configfile'], testvars.bad_client_config.format(HOST)) + self.write_config(self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices')) + self.invoke_runner_alt(hosts='http://127.0.0.1:9200', loglevel='DEBUG', logformat='ecs', logfile=self.args['configfile']) + assert 0 == self.result.exit_code + def test_cli_unreachable_cloud_id(self): + self.create_indices(10) + self.write_config(self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices')) + self.invoke_runner_alt(hosts='http://127.0.0.2:9200', cloud_id='abc:def', username='user', password='pass') + assert 1 == self.result.exit_code def test_no_config(self): # This test checks whether localhost:9200 is provided if no hosts or # port are in the configuration. But in testing, sometimes