From c75404a3d8e0a2c07e7e02da78e186998e31a890 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Tue, 2 Apr 2024 20:05:01 -0600 Subject: [PATCH] Release prep for 8.0.14 (#1710) * WIP branch for debugging cold2frozen weirdness * Release prep for 8.0.14 **Announcement** * A long awaited feature has been added, stealthily. It's fully in the documentation, but I do not yet plan to make a big announcement about it. In actions that search through indices, you can now specify a ``search_pattern`` to limit the number of indices that will be filtered. If no search pattern is specified, the behavior will be the same as it ever was: it will search through ``_all`` indices. The actions that support this option are: allocation, close, cold2frozen, delete_indices, forcemerge, index_settings, open, replicas, shrink, and snapshot. **Bugfix** * A mixup with naming conventions from the PII redacter tool got in the way of the cold2frozen action completing properly. **Changes** * Version bump: ``es_client==8.13.0`` * With the version bump to ``es_client`` comes a necessary change to calls to create a repository. In https://github.com/elastic/elasticsearch-specification/pull/2255 it became clear that using ``type`` and ``settings`` as it has been was insufficient for repository settings, so we go back to using a request ``body`` as in older times. This change affects ``esrepomgr`` in one place, and otherwise only in snapshot/restore testing. * Added the curator.helpers.getters.meta_getter to reduce near duplicate functions. * Changed curator.helpers.getters.get_indices to use the _cat API to pull indices. The primary driver for this is that it avoids pulling in the full mapping and index settings when all we really need to return is a list of index names. This should help keep memory from ballooning quite as much. The function also now allows for a search_pattern kwarg to search only for indices matching a pattern. This will also potentially make the initial index return list much smaller, and the list of indices needing to be filtered that much smaller. * Tests were added to ensure that the changes for ``get_indices`` work everywhere. * Tests were added to ensure that the new ``search_pattern`` did not break anything, and does behave as expected. --- curator/_version.py | 2 +- curator/actions/alias.py | 4 +- curator/actions/cold2frozen.py | 16 +++-- curator/cli.py | 11 ++- curator/cli_singletons/allocation.py | 3 + curator/cli_singletons/close.py | 6 +- curator/cli_singletons/delete.py | 5 +- curator/cli_singletons/forcemerge.py | 6 +- curator/cli_singletons/object_class.py | 6 ++ curator/cli_singletons/open_indices.py | 5 +- curator/cli_singletons/replicas.py | 6 +- curator/cli_singletons/show.py | 7 +- curator/cli_singletons/shrink.py | 6 +- curator/cli_singletons/snapshot.py | 4 +- curator/defaults/option_defaults.py | 6 ++ curator/helpers/getters.py | 86 +++++++++++++--------- curator/indexlist.py | 10 +-- curator/repomgrcli.py | 9 ++- curator/validators/options.py | 16 ++++- docs/Changelog.rst | 36 ++++++++++ docs/asciidoc/actions.asciidoc | 10 +++ docs/asciidoc/index.asciidoc | 8 +-- docs/asciidoc/options.asciidoc | 25 +++++++ docs/conf.py | 4 +- pyproject.toml | 2 +- tests/integration/__init__.py | 7 +- tests/integration/test_integrations.py | 10 +++ tests/unit/test_action_cold2frozen.py | 9 +-- tests/unit/test_action_restore.py | 3 + tests/unit/test_class_index_list.py | 2 +- tests/unit/test_helpers_getters.py | 98 ++++++++++---------------- 31 files changed, 291 insertions(+), 137 deletions(-) diff --git a/curator/_version.py b/curator/_version.py index c89fcf30..5d07530e 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.13' +__version__ = '8.0.14' diff --git a/curator/actions/alias.py b/curator/actions/alias.py index 901282d0..6dce0a67 100644 --- a/curator/actions/alias.py +++ b/curator/actions/alias.py @@ -47,7 +47,7 @@ def add(self, ilo, warn_if_no_indices=False): :type ilo: :py:class:`~.curator.indexlist.IndexList` """ verify_index_list(ilo) - self.loggit.debug('ADD -> ILO = %s', ilo) + self.loggit.debug('ADD -> ILO = %s', ilo.indices) if not self.client: self.client = ilo.client self.name = parse_datemath(self.client, self.name) @@ -80,7 +80,7 @@ def remove(self, ilo, warn_if_no_indices=False): :type ilo: :py:class:`~.curator.indexlist.IndexList` """ verify_index_list(ilo) - self.loggit.debug('REMOVE -> ILO = %s', ilo) + self.loggit.debug('REMOVE -> ILO = %s', ilo.indices) if not self.client: self.client = ilo.client self.name = parse_datemath(self.client, self.name) diff --git a/curator/actions/cold2frozen.py b/curator/actions/cold2frozen.py index 20daaf90..1d0e2743 100644 --- a/curator/actions/cold2frozen.py +++ b/curator/actions/cold2frozen.py @@ -1,6 +1,6 @@ """Snapshot and Restore action classes""" import logging -from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference +from curator.helpers.getters import get_alias_actions, get_tier_preference, meta_getter from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list from curator.helpers.utils import report_failure from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException @@ -76,7 +76,8 @@ def action_generator(self): :rtype: dict """ for idx in self.index_list.indices: - idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index'] + idx_settings = meta_getter(self.client, idx, get='settings') + self.loggit.debug('Index %s has settings: %s', idx, idx_settings) if has_lifecycle_name(idx_settings): self.loggit.critical( 'Index %s is associated with an ILM policy and this action will never work on ' @@ -85,13 +86,16 @@ def action_generator(self): if is_idx_partial(idx_settings): self.loggit.critical('Index %s is already in the frozen tier', idx) raise SearchableSnapshotException('Index is already in frozen tier') + snap = idx_settings['store']['snapshot']['snapshot_name'] snap_idx = idx_settings['store']['snapshot']['index_name'] repo = idx_settings['store']['snapshot']['repository_name'] - aliases = self.client.indices.get(index=idx)[idx]['aliases'] + msg = f'Index {idx} Snapshot name: {snap}, Snapshot index: {snap_idx}, repo: {repo}' + self.loggit.debug(msg) + + aliases = meta_getter(self.client, idx, get='alias') - prefix = get_frozen_prefix(snap_idx, idx) - renamed = f'{prefix}{snap_idx}' + renamed = f'partial-{idx}' if not self.index_settings: self.index_settings = { @@ -187,7 +191,7 @@ def cleanup(self, current_idx, newidx): def do_action(self): """ Do the actions outlined: - + Extract values from generated kwargs Mount Verify Update Aliases diff --git a/curator/cli.py b/curator/cli.py index b7c7f244..1a2f923b 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -86,14 +86,22 @@ def process_action(client, action_def, dry_run=False): logger = logging.getLogger(__name__) logger.debug('Configuration dictionary: %s', action_def.action_dict) mykwargs = {} + search_pattern = '_all' + logger.critical('INITIAL Action kwargs: %s', mykwargs) # Add some settings to mykwargs... if action_def.action == 'delete_indices': mykwargs['master_timeout'] = 30 ### Update the defaults with whatever came with opts, minus any Nones mykwargs.update(prune_nones(action_def.options)) + + # Pop out the search_pattern option, if present. + if 'search_pattern' in mykwargs: + search_pattern = mykwargs.pop('search_pattern') + logger.debug('Action kwargs: %s', mykwargs) + logger.critical('Post search_pattern Action kwargs: %s', mykwargs) ### Set up the action ### logger.debug('Running "%s"', action_def.action.upper()) @@ -120,8 +128,9 @@ def process_action(client, action_def, dry_run=False): mykwargs.pop('repository') # We don't need to send this value to the action action_def.instantiate('list_obj', client, repository=action_def.options['repository']) else: - action_def.instantiate('list_obj', client) + action_def.instantiate('list_obj', client, search_pattern=search_pattern) action_def.list_obj.iterate_filters({'filters': action_def.filters}) + logger.critical('Pre Instantiation Action kwargs: %s', mykwargs) action_def.instantiate('action_cls', action_def.list_obj, **mykwargs) ### Do the action if dry_run: diff --git a/curator/cli_singletons/allocation.py b/curator/cli_singletons/allocation.py index 260d8a65..c8a87db3 100644 --- a/curator/cli_singletons/allocation.py +++ b/curator/cli_singletons/allocation.py @@ -4,6 +4,7 @@ from curator.cli_singletons.utils import validate_filter_json @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--key', type=str, required=True, help='Node identification tag') @click.option('--value', type=str, default=None, help='Value associated with --key') @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude'])) @@ -47,6 +48,7 @@ @click.pass_context def allocation( ctx, + search_pattern, key, value, allocation_type, @@ -61,6 +63,7 @@ def allocation( Shard Routing Allocation """ manual_options = { + 'search_pattern': search_pattern, 'key': key, 'value': value, 'allocation_type': allocation_type, diff --git a/curator/cli_singletons/close.py b/curator/cli_singletons/close.py index c6d11d3f..e4f9ea37 100644 --- a/curator/cli_singletons/close.py +++ b/curator/cli_singletons/close.py @@ -4,6 +4,7 @@ from curator.cli_singletons.utils import validate_filter_json @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed') @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed') @click.option( @@ -24,11 +25,14 @@ required=True ) @click.pass_context -def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, filter_list): +def close( + ctx, search_pattern, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, + filter_list): """ Close Indices """ manual_options = { + 'search_pattern': search_pattern, 'skip_flush': skip_flush, 'delete_aliases': delete_aliases, 'allow_ilm_indices': allow_ilm_indices, diff --git a/curator/cli_singletons/delete.py b/curator/cli_singletons/delete.py index e43546d3..6c788a9b 100644 --- a/curator/cli_singletons/delete.py +++ b/curator/cli_singletons/delete.py @@ -5,6 +5,7 @@ #### Indices #### @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option( '--ignore_empty_list', is_flag=True, @@ -23,7 +24,7 @@ required=True ) @click.pass_context -def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): +def delete_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list): """ Delete Indices """ @@ -31,7 +32,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): action = CLIAction( 'delete_indices', ctx.obj['configdict'], - {'allow_ilm_indices':allow_ilm_indices}, + {'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list ) diff --git a/curator/cli_singletons/forcemerge.py b/curator/cli_singletons/forcemerge.py index ed6cb5bd..d62dc800 100644 --- a/curator/cli_singletons/forcemerge.py +++ b/curator/cli_singletons/forcemerge.py @@ -4,6 +4,7 @@ from curator.cli_singletons.utils import validate_filter_json @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option( '--max_num_segments', type=int, @@ -32,11 +33,14 @@ help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list): +def forcemerge( + ctx, search_pattern, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, + filter_list): """ forceMerge Indices (reduce segment count) """ manual_options = { + 'search_pattern': search_pattern, 'max_num_segments': max_num_segments, 'delay': delay, 'allow_ilm_indices': allow_ilm_indices, diff --git a/curator/cli_singletons/object_class.py b/curator/cli_singletons/object_class.py index 900282e3..26e7be0c 100644 --- a/curator/cli_singletons/object_class.py +++ b/curator/cli_singletons/object_class.py @@ -76,6 +76,12 @@ def __init__(self, action, configdict, option_dict, filter_list, ignore_empty_li self.check_options(option_dict) else: self.options = option_dict + + # Pop out search_pattern if it's there + self.search_pattern = '_all' + if 'search_pattern' in self.options: + self.search_pattern = self.options.pop('search_pattern') + # Extract allow_ilm_indices so it can be handled separately. if 'allow_ilm_indices' in self.options: self.allow_ilm = self.options.pop('allow_ilm_indices') diff --git a/curator/cli_singletons/open_indices.py b/curator/cli_singletons/open_indices.py index 61faa31d..8eee0836 100644 --- a/curator/cli_singletons/open_indices.py +++ b/curator/cli_singletons/open_indices.py @@ -4,6 +4,7 @@ from curator.cli_singletons.utils import validate_filter_json @click.command(name='open') +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option( '--ignore_empty_list', is_flag=True, @@ -22,7 +23,7 @@ required=True ) @click.pass_context -def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): +def open_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list): """ Open Indices """ @@ -30,7 +31,7 @@ def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): action = CLIAction( ctx.info_name, ctx.obj['configdict'], - {'allow_ilm_indices':allow_ilm_indices}, + {'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list ) diff --git a/curator/cli_singletons/replicas.py b/curator/cli_singletons/replicas.py index 70d039f0..e86a3a03 100644 --- a/curator/cli_singletons/replicas.py +++ b/curator/cli_singletons/replicas.py @@ -4,6 +4,7 @@ from curator.cli_singletons.utils import validate_filter_json @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') @click.option( '--wait_for_completion/--no-wait_for_completion', @@ -29,11 +30,14 @@ required=True ) @click.pass_context -def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list): +def replicas( + ctx, search_pattern, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, + filter_list): """ Change Replica Count """ manual_options = { + 'search_pattern': search_pattern, 'count': count, 'wait_for_completion': wait_for_completion, 'allow_ilm_indices': allow_ilm_indices, diff --git a/curator/cli_singletons/show.py b/curator/cli_singletons/show.py index 3751b0f7..8f11724c 100644 --- a/curator/cli_singletons/show.py +++ b/curator/cli_singletons/show.py @@ -11,6 +11,7 @@ # pylint: disable=line-too-long @click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True) @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True) @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True) @@ -18,7 +19,9 @@ @click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.') @click.pass_context -def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, filter_list): +def show_indices( + ctx, search_pattern, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, + filter_list): """ Show Indices """ @@ -26,7 +29,7 @@ def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indic action = CLIAction( 'show_indices', ctx.obj['configdict'], - {'allow_ilm_indices': allow_ilm_indices}, + {'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices}, filter_list, ignore_empty_list ) diff --git a/curator/cli_singletons/shrink.py b/curator/cli_singletons/shrink.py index 79653143..64943038 100644 --- a/curator/cli_singletons/shrink.py +++ b/curator/cli_singletons/shrink.py @@ -5,6 +5,7 @@ # pylint: disable=line-too-long @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True) @click.option('--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict) @click.option('--number_of_shards', default=1, type=int, help='Shrink to this many shards per index') @@ -25,8 +26,8 @@ @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context def shrink( - ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix, - shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings, + ctx, search_pattern, shrink_node, node_filters, number_of_shards, number_of_replicas, + shrink_prefix, shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings, wait_for_active_shards, wait_for_rebalance, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, filter_list ): @@ -34,6 +35,7 @@ def shrink( Shrink Indices to --number_of_shards """ manual_options = { + 'search_pattern': search_pattern, 'shrink_node': shrink_node, 'node_filters': node_filters, 'number_of_shards': number_of_shards, diff --git a/curator/cli_singletons/snapshot.py b/curator/cli_singletons/snapshot.py index ad0e69fc..f1cef5ed 100644 --- a/curator/cli_singletons/snapshot.py +++ b/curator/cli_singletons/snapshot.py @@ -5,6 +5,7 @@ # pylint: disable=line-too-long @click.command() +@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S') @click.option('--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.') @@ -19,7 +20,7 @@ @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context def snapshot( - ctx, repository, name, ignore_unavailable, include_global_state, partial, + ctx, search_pattern, repository, name, ignore_unavailable, include_global_state, partial, skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, filter_list ): @@ -27,6 +28,7 @@ def snapshot( Snapshot Indices """ manual_options = { + 'search_pattern': search_pattern, 'name': name, 'repository': repository, 'ignore_unavailable': ignore_unavailable, diff --git a/curator/defaults/option_defaults.py b/curator/defaults/option_defaults.py index 31a0d637..ef4c3018 100644 --- a/curator/defaults/option_defaults.py +++ b/curator/defaults/option_defaults.py @@ -380,6 +380,12 @@ def cluster_routing_value(): """ return {Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')} +def search_pattern(): + """ + :returns: ``{Optional('search_pattern', default='_all'): Any(*string_types)}`` + """ + return {Optional('search_pattern', default='_all'): Any(*string_types)} + def shrink_node(): """ :returns: ``{Required('shrink_node'): Any(*string_types)}`` diff --git a/curator/helpers/getters.py b/curator/helpers/getters.py index 7bbad6e7..6ef1ade3 100644 --- a/curator/helpers/getters.py +++ b/curator/helpers/getters.py @@ -2,7 +2,8 @@ import logging import re from elasticsearch8 import exceptions as es8exc -from curator.exceptions import CuratorException, FailedExecution, MissingArgument +from curator.exceptions import ( + ConfigurationError, CuratorException, FailedExecution, MissingArgument) def byte_size(num, suffix='B'): """ @@ -65,35 +66,9 @@ def role_check(role, node_info): retval[role] = True return retval -def get_frozen_prefix(oldidx, curridx): +def get_indices(client, search_pattern='_all'): """ - Use regular expression magic to extract the prefix from the current index, and then use - that with ``partial-`` in front to name the resulting index. - - If there is no prefix, then we just send back ``partial-`` - - :param oldidx: The index name before it was mounted in cold tier - :param curridx: The current name of the index, as mounted in cold tier - - :type oldidx: str - :type curridx: str - - :returns: The prefix to prepend the index name with for mounting as frozen - :rtype: str - """ - logger = logging.getLogger(__name__) - pattern = f'^(.*){oldidx}$' - regexp = re.compile(pattern) - match = regexp.match(curridx) - prefix = match.group(1) - logger.debug('Detected match group for prefix: %s', prefix) - if not prefix: - return 'partial-' - return f'partial-{prefix}' - -def get_indices(client): - """ - Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` + Calls :py:meth:`~.elasticsearch.client.CatClient.indices` :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` @@ -102,12 +77,20 @@ def get_indices(client): :rtype: list """ logger = logging.getLogger(__name__) + indices = [] try: - indices = list(client.indices.get_settings(index='*', expand_wildcards='open,closed')) - logger.debug('All indices: %s', indices) - return indices + # Doing this in two stages because IndexList also calls for these args, and the unit tests + # need to Mock this call the same exact way. + resp = client.cat.indices( + index=search_pattern, expand_wildcards='open,closed', h='index,status', format='json') except Exception as err: raise FailedExecution(f'Failed to get indices. Error: {err}') from err + if not resp: + return indices + for entry in resp: + indices.append(entry['index']) + logger.debug('All indices: %s', indices) + return indices def get_repository(client, repository=''): """ @@ -275,6 +258,45 @@ def index_size(client, idx, value='total'): """ return client.indices.stats(index=idx)['indices'][idx][value]['store']['size_in_bytes'] +def meta_getter(client, idx, get=None): + """Meta Getter + Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` or + :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` + + :param client: A client connection object + :param idx: An Elasticsearch index + :param get: The kind of get to perform, e.g. settings or alias + + :type client: :py:class:`~.elasticsearch.Elasticsearch` + :type idx: str + :type get: str + + :returns: The settings from the get call to the named index + :rtype: dict + """ + logger = logging.getLogger(__name__) + acceptable = ['settings', 'alias'] + if not get: + raise ConfigurationError('"get" can not be a NoneType') + if get not in acceptable: + raise ConfigurationError(f'"get" must be one of {acceptable}') + retval = {} + try: + if get == 'settings': + retval = client.indices.get_settings(index=idx)[idx]['settings']['index'] + elif get == 'alias': + retval = client.indices.get_alias(index=idx)[idx]['aliases'] + except es8exc.NotFoundError as missing: + logger.error('Index %s was not found!', idx) + raise es8exc.NotFoundError from missing + except KeyError as err: + logger.error('Key not found: %s', err) + raise KeyError from err + # pylint: disable=broad-except + except Exception as exc: + logger.error('Exception encountered: %s', exc) + return retval + def name_to_node_id(client, name): """ Calls :py:meth:`~.elasticsearch.client.NodesClient.stats` diff --git a/curator/indexlist.py b/curator/indexlist.py index 352d647f..51b7ac57 100644 --- a/curator/indexlist.py +++ b/curator/indexlist.py @@ -18,7 +18,7 @@ class IndexList: """IndexList class""" - def __init__(self, client): + def __init__(self, client, search_pattern='_all'): verify_client_object(client) self.loggit = logging.getLogger('curator.indexlist') #: An :py:class:`~.elasticsearch.Elasticsearch` client object passed from param ``client`` @@ -34,7 +34,7 @@ def __init__(self, client): #: All indices in the cluster at instance creation time. #: **Type:** :py:class:`list` self.all_indices = [] - self.__get_indices() + self.__get_indices(search_pattern) self.age_keyfield = None def __actionable(self, idx): @@ -62,12 +62,12 @@ def __excludify(self, condition, exclude, index, msg=None): if msg: self.loggit.debug('%s: %s', text, msg) - def __get_indices(self): + def __get_indices(self, pattern): """ Pull all indices into ``all_indices``, then populate ``indices`` and ``index_info`` """ - self.loggit.debug('Getting all indices') - self.all_indices = get_indices(self.client) + self.loggit.debug('Getting indices matching search_pattern: "%s"', pattern) + self.all_indices = get_indices(self.client, search_pattern=pattern) self.indices = self.all_indices[:] # if self.indices: # for index in self.indices: diff --git a/curator/repomgrcli.py b/curator/repomgrcli.py index c2c12f36..feca9765 100644 --- a/curator/repomgrcli.py +++ b/curator/repomgrcli.py @@ -86,10 +86,13 @@ def create_repo(ctx, repo_name=None, repo_type=None, repo_settings=None, verify= :rtype: None """ logger = logging.getLogger('curator.repomgrcli.create_repo') - esclient = get_client(ctx) + client = get_client(ctx) + request_body = { + 'type': repo_type, + 'settings': repo_settings + } try: - esclient.snapshot.create_repository( - name=repo_name, settings=repo_settings, type=repo_type, verify=verify) + client.snapshot.create_repository(name=repo_name, body=request_body, verify=verify) except ApiError as exc: if exc.meta.status >= 500: logger.critical('Server-side error!') diff --git a/curator/validators/options.py b/curator/validators/options.py index d6a057bb..cca0e370 100644 --- a/curator/validators/options.py +++ b/curator/validators/options.py @@ -22,6 +22,7 @@ def action_specific(action): option_defaults.extra_settings(), ], 'allocation' : [ + option_defaults.search_pattern(), option_defaults.key(), option_defaults.value(), option_defaults.allocation_type(), @@ -30,6 +31,7 @@ def action_specific(action): option_defaults.max_wait(action), ], 'close' : [ + option_defaults.search_pattern(), option_defaults.delete_aliases(), option_defaults.skip_flush(), ], @@ -42,6 +44,7 @@ def action_specific(action): option_defaults.max_wait(action), ], 'cold2frozen' : [ + option_defaults.search_pattern(), option_defaults.c2f_index_settings(), option_defaults.c2f_ignore_index_settings(), option_defaults.wait_for_completion('cold2frozen'), @@ -51,22 +54,28 @@ def action_specific(action): option_defaults.ignore_existing(), option_defaults.extra_settings(), ], - 'delete_indices' : [], + 'delete_indices' : [ + option_defaults.search_pattern(), + ], 'delete_snapshots' : [ option_defaults.repository(), option_defaults.retry_interval(), option_defaults.retry_count(), ], 'forcemerge' : [ + option_defaults.search_pattern(), option_defaults.delay(), option_defaults.max_num_segments(), ], 'index_settings' : [ + option_defaults.search_pattern(), option_defaults.index_settings(), option_defaults.ignore_unavailable(), option_defaults.preserve_existing(), ], - 'open' : [], + 'open' : [ + option_defaults.search_pattern(), + ], 'reindex' : [ option_defaults.request_body(), option_defaults.refresh(), @@ -85,6 +94,7 @@ def action_specific(action): option_defaults.migration_suffix(), ], 'replicas' : [ + option_defaults.search_pattern(), option_defaults.count(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), @@ -114,6 +124,7 @@ def action_specific(action): option_defaults.skip_repo_fs_check(), ], 'snapshot' : [ + option_defaults.search_pattern(), option_defaults.repository(), option_defaults.name(action), option_defaults.ignore_unavailable(), @@ -125,6 +136,7 @@ def action_specific(action): option_defaults.skip_repo_fs_check(), ], 'shrink' : [ + option_defaults.search_pattern(), option_defaults.shrink_node(), option_defaults.node_filters(), option_defaults.number_of_shards(), diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 1c62d7cf..f5a40ef2 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,42 @@ Changelog ========= +8.0.14 (2 April 2024) +--------------------- + +**Announcement** + + * A long awaited feature has been added, stealthily. It's fully in the documentation, but I do + not yet plan to make a big announcement about it. In actions that search through indices, you + can now specify a ``search_pattern`` to limit the number of indices that will be filtered. If + no search pattern is specified, the behavior will be the same as it ever was: it will search + through ``_all`` indices. The actions that support this option are: allocation, close, + cold2frozen, delete_indices, forcemerge, index_settings, open, replicas, shrink, and snapshot. + +**Bugfix** + + * A mixup with naming conventions from the PII redacter tool got in the way of the cold2frozen + action completing properly. + +**Changes** + + * Version bump: ``es_client==8.13.0`` + * With the version bump to ``es_client`` comes a necessary change to calls to create a + repository. In https://github.com/elastic/elasticsearch-specification/pull/2255 it became + clear that using ``type`` and ``settings`` as it has been was insufficient for repository + settings, so we go back to using a request ``body`` as in older times. This change affects + ``esrepomgr`` in one place, and otherwise only in snapshot/restore testing. + * Added the curator.helpers.getters.meta_getter to reduce near duplicate functions. + * Changed curator.helpers.getters.get_indices to use the _cat API to pull indices. The primary + driver for this is that it avoids pulling in the full mapping and index settings when all we + really need to return is a list of index names. This should help keep memory from ballooning + quite as much. The function also now allows for a search_pattern kwarg to search only for + indices matching a pattern. This will also potentially make the initial index return list much + smaller, and the list of indices needing to be filtered that much smaller. + * Tests were added to ensure that the changes for ``get_indices`` work everywhere. + * Tests were added to ensure that the new ``search_pattern`` did not break anything, and does + behave as expected. + 8.0.13 (26 March 2024) ---------------------- diff --git a/docs/asciidoc/actions.asciidoc b/docs/asciidoc/actions.asciidoc index cb61d7d0..47537355 100644 --- a/docs/asciidoc/actions.asciidoc +++ b/docs/asciidoc/actions.asciidoc @@ -153,6 +153,7 @@ defined by `wait_interval`. === Optional settings +* <> * <> * <> * <> @@ -191,6 +192,7 @@ aliases beforehand. === Optional settings +* <> * <> * <> * <> @@ -347,6 +349,7 @@ NOTE: If unset, the default behavior is to ensure that the `index.refresh_interv === Optional settings +* <> * <> * <> * <> @@ -511,6 +514,7 @@ filters: === Optional settings +* <> * <> * <> * <> @@ -613,6 +617,7 @@ filters: === Optional settings +* <> * <> * <> * <> @@ -683,6 +688,7 @@ settings, and to be able to verify configurational integrity in the YAML file, === Optional settings +* <> * <> * <> * <> @@ -714,6 +720,7 @@ This action opens the selected indices. === Optional settings +* <> * <> * <> * <> @@ -836,6 +843,7 @@ defined by `wait_interval`. === Optional settings +* <> * <> * <> * <> @@ -1174,6 +1182,7 @@ as needed. === Optional settings +* <> * <> * <> * <> @@ -1233,6 +1242,7 @@ read about them and change them accordingly. === Optional settings +* <> * <> * <> * <> diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index 76cae774..945c1d80 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,9 +1,9 @@ -:curator_version: 8.0.13 +:curator_version: 8.0.14 :curator_major: 8 :curator_doc_tree: 8.0 -:es_py_version: 8.12.1 -:es_doc_tree: 8.12 -:stack_doc_tree: 8.12 +:es_py_version: 8.13.0 +:es_doc_tree: 8.13 +:stack_doc_tree: 8.13 :pybuild_ver: 3.11.7 :copyright_years: 2011-2024 :ref: http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree} diff --git a/docs/asciidoc/options.asciidoc b/docs/asciidoc/options.asciidoc index 86783c5f..15dcbc3a 100644 --- a/docs/asciidoc/options.asciidoc +++ b/docs/asciidoc/options.asciidoc @@ -48,6 +48,7 @@ Options are settings used by <>. * <> * <> * <> +* <> * <> * <> * <> @@ -2151,6 +2152,30 @@ The value of this setting must be either `allocation` or `rebalance` There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. +[[option_search_pattern]] +== search_pattern + +NOTE: This setting is only used by the + <>, <>, <>, <>, <>, + <>, <>, <>, <>, and <> actions. + +[source,yaml] +------------- +action: delete_indices +description: "Delete selected indices" +options: + search_pattern: 'index-*' +filters: +- filtertype: ... +------------- + +The value of this option can be a comma-separated list of data streams, indices, and aliases used +to limit the request. Supports wildcards (*). To target all data streams and indices, omit this +parameter or use * or _all. If using wildcards it is highly recommended to encapsulate the entire +search pattern in single quotes, e.g. `search_pattern: 'a*,b*,c*'` + +The default value is `_all`. + [[option_setting]] == setting diff --git a/docs/conf.py b/docs/conf.py index 985b7d9d..8b99f99e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,8 +72,8 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3.11', None), - 'es_client': ('https://es-client.readthedocs.io/en/v8.12.9', None), - 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.12.1', None), + 'es_client': ('https://es-client.readthedocs.io/en/v8.13.0', None), + 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.13.0', None), 'voluptuous': ('http://alecthomas.github.io/voluptuous/docs/_build/html', None), 'click': ('https://click.palletsprojects.com/en/8.1.x', None), } diff --git a/pyproject.toml b/pyproject.toml index e81f9969..57207335 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ keywords = [ 'index-expiry' ] dependencies = [ - "es_client==8.12.9" + "es_client==8.13.0" ] [project.optional-dependencies] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 50cd3579..47a9b7a8 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -178,8 +178,11 @@ def delete_snapshot(self, name): ) def create_repository(self): - args = {'location': self.args['location']} - self.client.snapshot.create_repository(name=self.args['repository'], type='fs', settings=args) + request_body = { + 'type': 'fs', + 'settings': {'location': self.args['location']} + } + self.client.snapshot.create_repository(name=self.args['repository'], body=request_body) def delete_repositories(self): result = self.client.snapshot.get_repository(name='*') diff --git a/tests/integration/test_integrations.py b/tests/integration/test_integrations.py index 6a26c8dd..62e57e12 100644 --- a/tests/integration/test_integrations.py +++ b/tests/integration/test_integrations.py @@ -149,3 +149,13 @@ def test_not_needful(self): ilo.get_index_stats() ilo.get_index_stats() # This time index_info is already populated and it will skip assert ilo.index_info[self.IDX1][key] == 0 + def test_search_pattern_1(self): + """Check to see if providing a search_pattern limits the index list""" + pattern = 'd*' + self.create_index(self.IDX1) + self.create_index(self.IDX2) + self.create_index(self.IDX3) + ilo1 = IndexList(self.client) + ilo2 = IndexList(self.client, search_pattern=pattern) + assert ilo1.indices == [self.IDX1, self.IDX2, self.IDX3] + assert ilo2.indices == [self.IDX1, self.IDX2] diff --git a/tests/unit/test_action_cold2frozen.py b/tests/unit/test_action_cold2frozen.py index becda8c4..c19218ad 100644 --- a/tests/unit/test_action_cold2frozen.py +++ b/tests/unit/test_action_cold2frozen.py @@ -38,7 +38,6 @@ def test_action_generator1(self): settings_ss = { testvars.named_index: { 'aliases': {'my_alias': {}}, - 'mappings': {}, 'settings': { 'index': { 'creation_date': '1456963200172', @@ -58,7 +57,9 @@ def test_action_generator1(self): } } } - self.client.indices.get.return_value = settings_ss + + self.client.indices.get_settings.return_value = settings_ss + self.client.indices.get_alias.return_value = settings_ss roles = ['data_content'] self.client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} c2f = Cold2Frozen(self.ilo) @@ -92,7 +93,7 @@ def test_action_generator2(self): 'settings': {'index': {'lifecycle': {'name': 'guaranteed_fail'}}} } } - self.client.indices.get.return_value = settings_ss + self.client.indices.get_settings.return_value = settings_ss c2f = Cold2Frozen(self.ilo) with pytest.raises(CuratorException, match='associated with an ILM policy'): for result in c2f.action_generator(): @@ -110,7 +111,7 @@ def test_action_generator3(self): } } } - self.client.indices.get.return_value = settings_ss + self.client.indices.get_settings.return_value = settings_ss c2f = Cold2Frozen(self.ilo) with pytest.raises(SearchableSnapshotException, match='Index is already in frozen tier'): for result in c2f.action_generator(): diff --git a/tests/unit/test_action_restore.py b/tests/unit/test_action_restore.py index ecd5702f..76e75fd9 100644 --- a/tests/unit/test_action_restore.py +++ b/tests/unit/test_action_restore.py @@ -100,6 +100,7 @@ def test_report_state_all(self): client.info.return_value = {'version': {'number': '5.0.0'} } client.snapshot.get.return_value = testvars.snapshot client.snapshot.get_repository.return_value = testvars.test_repo + client.cat.indices.return_value = testvars.state_named client.indices.get_settings.return_value = testvars.settings_named slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) @@ -109,6 +110,7 @@ def test_report_state_not_all(self): client.info.return_value = {'version': {'number': '5.0.0'} } client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo + client.cat.indices.return_value = testvars.state_one client.indices.get_settings.return_value = testvars.settings_one slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore( @@ -121,6 +123,7 @@ def test_do_action_success(self): client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes + client.cat.indices.return_value = testvars.state_named client.indices.get_settings.return_value = testvars.settings_named client.indices.recovery.return_value = testvars.recovery_output slo = SnapshotList(client, repository=testvars.repo_name) diff --git a/tests/unit/test_class_index_list.py b/tests/unit/test_class_index_list.py index 77e79a85..93645491 100644 --- a/tests/unit/test_class_index_list.py +++ b/tests/unit/test_class_index_list.py @@ -52,7 +52,7 @@ def test_init_bad_client(self): self.assertRaises(TypeError, IndexList, client) def test_init_get_indices_exception(self): self.builder() - self.client.indices.get_settings.side_effect = testvars.fake_fail + self.client.cat.indices.side_effect = testvars.fake_fail self.assertRaises(FailedExecution, IndexList, self.client) def test_init(self): self.builder() diff --git a/tests/unit/test_helpers_getters.py b/tests/unit/test_helpers_getters.py index 9c3efb19..ffc9ec16 100644 --- a/tests/unit/test_helpers_getters.py +++ b/tests/unit/test_helpers_getters.py @@ -5,11 +5,7 @@ from elastic_transport import ApiResponseMeta from elasticsearch8 import NotFoundError, TransportError from curator.exceptions import CuratorException, FailedExecution, MissingArgument -from curator.helpers.getters import ( - byte_size, get_alias_actions, get_frozen_prefix, get_indices, get_repository, get_snapshot, - get_snapshot_data, get_tier_preference, name_to_node_id, node_id_to_name, node_roles, - single_data_path -) +from curator.helpers import getters FAKE_FAIL = Exception('Simulated Failure') NAMED_INDICES = [ "index-2015.01.01", "index-2015.02.01" ] @@ -33,7 +29,7 @@ def test_byte_size(self): size = 3*1024*1024*1024*1024*1024*1024*1024 unit = ['Z','E','P','T','G','M','K',''] for i in range(0,7): - assert f'3.0{unit[i]}B' == byte_size(size) + assert f'3.0{unit[i]}B' == getters.byte_size(size) size /= 1024 def test_byte_size_yotta(self): """test_byte_size_yotta @@ -41,14 +37,14 @@ def test_byte_size_yotta(self): Output should match expected """ size = 3*1024*1024*1024*1024*1024*1024*1024*1024 - assert '3.0YB' == byte_size(size) + assert '3.0YB' == getters.byte_size(size) def test_raise_invalid(self): """test_raise_invalid Should raise a TypeError exception if an invalid value is passed """ with pytest.raises(TypeError): - byte_size('invalid') + getters.byte_size('invalid') class TestGetIndices(TestCase): """TestGetIndices @@ -57,36 +53,36 @@ class TestGetIndices(TestCase): """ IDX1 = 'index-2016.03.03' IDX2 = 'index-2016.03.04' - SETTINGS = { - IDX1: {'state': 'open'}, - IDX2: {'state': 'open'} - } + RESPONSE = [ + {'index': IDX1, 'state': 'open'}, + {'index': IDX2, 'state': 'open'} + ] def test_client_exception(self): """test_client_exception Should raise a FailedExecution exception when an upstream exception occurs """ client = Mock() - client.indices.get_settings.return_value = self.SETTINGS - client.indices.get_settings.side_effect = FAKE_FAIL + client.cat.indices.return_value = self.RESPONSE + client.cat.indices.side_effect = FAKE_FAIL with pytest.raises(FailedExecution): - get_indices(client) + getters.get_indices(client) def test_positive(self): """test_positive Output should match expected """ client = Mock() - client.indices.get_settings.return_value = self.SETTINGS - self.assertEqual([self.IDX1, self.IDX2], sorted(get_indices(client))) + client.cat.indices.return_value = self.RESPONSE + self.assertEqual([self.IDX1, self.IDX2], sorted(getters.get_indices(client))) def test_empty(self): """test_empty Output should be an empty list """ client = Mock() - client.indices.get_settings.return_value = {} - self.assertEqual([], get_indices(client)) + client.cat.indices.return_value = {} + self.assertEqual([], getters.get_indices(client)) class TestGetRepository(TestCase): """TestGetRepository @@ -101,7 +97,7 @@ def test_get_repository_missing_arg(self): """ client = Mock() client.snapshot.get_repository.return_value = {} - assert not get_repository(client) + assert not getters.get_repository(client) def test_get_repository_positive(self): """test_get_repository_positive @@ -109,7 +105,7 @@ def test_get_repository_positive(self): """ client = Mock() client.snapshot.get_repository.return_value = TEST_REPO - assert TEST_REPO == get_repository(client, repository=REPO_NAME) + assert TEST_REPO == getters.get_repository(client, repository=REPO_NAME) def test_get_repository_transporterror_negative(self): """test_get_repository_transporterror_negative @@ -118,7 +114,7 @@ def test_get_repository_transporterror_negative(self): client = Mock() client.snapshot.get_repository.side_effect = TransportError(503, ('exception', 'reason')) with pytest.raises(CuratorException, match=r'503 Check Elasticsearch logs'): - get_repository(client, repository=REPO_NAME) + getters.get_repository(client, repository=REPO_NAME) def test_get_repository_notfounderror_negative(self): """test_get_repository_notfounderror_negative @@ -133,7 +129,7 @@ def test_get_repository_notfounderror_negative(self): effect = NotFoundError(msg, meta, body) client.snapshot.get_repository.side_effect = effect with pytest.raises(CuratorException, match=r'Error: NotFoundError'): - get_repository(client, repository=REPO_NAME) + getters.get_repository(client, repository=REPO_NAME) def test_get_repository_all_positive(self): """test_get_repository_all_positive @@ -141,7 +137,7 @@ def test_get_repository_all_positive(self): """ client = Mock() client.snapshot.get_repository.return_value = self.MULTI - assert self.MULTI == get_repository(client) + assert self.MULTI == getters.get_repository(client) class TestGetSnapshot(TestCase): """TestGetSnapshot @@ -155,7 +151,7 @@ def test_get_snapshot_missing_repository_arg(self): """ client = Mock() with pytest.raises(MissingArgument, match=r'No value for "repository" provided'): - get_snapshot(client, snapshot=SNAP_NAME) + getters.get_snapshot(client, snapshot=SNAP_NAME) def test_get_snapshot_positive(self): """test_get_snapshot_positive @@ -163,7 +159,7 @@ def test_get_snapshot_positive(self): """ client = Mock() client.snapshot.get.return_value = SNAPSHOT - assert SNAPSHOT == get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) + assert SNAPSHOT == getters.get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) def test_get_snapshot_transporterror_negative(self): """test_get_snapshot_transporterror_negative @@ -173,7 +169,7 @@ def test_get_snapshot_transporterror_negative(self): client.snapshot.get_repository.return_value = TEST_REPO client.snapshot.get.side_effect = TransportError(401, "simulated error") with pytest.raises(FailedExecution, match=r'Error: 401'): - get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) + getters.get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) def test_get_snapshot_notfounderror_negative(self): """test_get_snapshot_notfounderror_negative @@ -185,7 +181,7 @@ def test_get_snapshot_notfounderror_negative(self): meta = ApiResponseMeta(404, '1.1', {}, 1.0, None) client.snapshot.get.side_effect = NotFoundError('simulated error', meta, 'simulated error') with pytest.raises(FailedExecution, match=r'Error: NotFoundError'): - get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) + getters.get_snapshot(client, repository=REPO_NAME, snapshot=SNAP_NAME) class TestGetSnapshotData(TestCase): """TestGetSnapshotData @@ -199,7 +195,7 @@ def test_missing_repo_arg(self): """ client = Mock() with pytest.raises(MissingArgument, match=r'No value for "repository" provided'): - get_snapshot_data(client) + getters.get_snapshot_data(client) def test_return_data(self): """test_return_data @@ -208,7 +204,7 @@ def test_return_data(self): client = Mock() client.snapshot.get.return_value = SNAPSHOTS client.snapshot.get_repository.return_value = TEST_REPO - assert SNAPSHOTS['snapshots'] == get_snapshot_data(client, repository=REPO_NAME) + assert SNAPSHOTS['snapshots'] == getters.get_snapshot_data(client, repository=REPO_NAME) def test_raises_exception_onfail(self): """test_raises_exception_onfail @@ -219,7 +215,7 @@ def test_raises_exception_onfail(self): client.snapshot.get.side_effect = TransportError(401, "simulated error") client.snapshot.get_repository.return_value = TEST_REPO with pytest.raises(FailedExecution, match=r'Error: 401'): - get_snapshot_data(client, repository=REPO_NAME) + getters.get_snapshot_data(client, repository=REPO_NAME) class TestNodeRoles(TestCase): """TestNodeRoles @@ -235,7 +231,7 @@ def test_node_roles(self): expected = ['data'] client = Mock() client.nodes.info.return_value = {'nodes':{node_id:{'roles': expected}}} - assert expected == node_roles(client, node_id) + assert expected == getters.node_roles(client, node_id) class TestSingleDataPath(TestCase): """TestSingleDataPath @@ -250,7 +246,7 @@ def test_single_data_path(self): node_id = 'my_node' client = Mock() client.nodes.stats.return_value = {'nodes':{node_id:{'fs':{'data':['one']}}}} - assert single_data_path(client, node_id) + assert getters.single_data_path(client, node_id) def test_two_data_paths(self): """test_two_data_paths @@ -259,7 +255,7 @@ def test_two_data_paths(self): node_id = 'my_node' client = Mock() client.nodes.stats.return_value = {'nodes':{node_id:{'fs':{'data':['one','two']}}}} - assert not single_data_path(client, node_id) + assert not getters.single_data_path(client, node_id) class TestNameToNodeId(TestCase): """TestNameToNodeId @@ -275,7 +271,7 @@ def test_positive(self): node_name = 'node_name' client = Mock() client.nodes.stats.return_value = {'nodes':{node_id:{'name':node_name}}} - assert node_id == name_to_node_id(client, node_name) + assert node_id == getters.name_to_node_id(client, node_name) def test_negative(self): """test_negative @@ -285,7 +281,7 @@ def test_negative(self): node_name = 'node_name' client = Mock() client.nodes.stats.return_value = {'nodes':{node_id:{'name':node_name}}} - assert None is name_to_node_id(client, 'wrong_name') + assert None is getters.name_to_node_id(client, 'wrong_name') class TestNodeIdToName(TestCase): """TestNodeIdToName @@ -299,7 +295,7 @@ def test_negative(self): """ client = Mock() client.nodes.stats.return_value = {'nodes':{'my_node_id':{'name':'my_node_name'}}} - assert None is node_id_to_name(client, 'not_my_node_id') + assert None is getters.node_id_to_name(client, 'not_my_node_id') class TestGetAliasActions(TestCase): """TestGetAliasActions @@ -320,23 +316,7 @@ def test_get_alias_actions(self): 'add': {'index': newidx, 'alias': name} } ] - assert get_alias_actions(oldidx, newidx, aliases) == expected - -class TestGetFrozenPrefix(TestCase): - """TestGetFrozenPrefix - - Test helpers.getters.get_frozen_prefix functionality. - """ - def test_get_frozen_prefix1(self): - """test_get_frozen_prefix1""" - oldidx = 'test-000001' - curridx = 'restored-test-000001' - assert get_frozen_prefix(oldidx, curridx) == 'partial-restored-' - def test_get_frozen_prefix2(self): - """test_get_frozen_prefix2""" - oldidx = 'test-000001' - curridx = 'test-000001' - assert get_frozen_prefix(oldidx, curridx) == 'partial-' + assert getters.get_alias_actions(oldidx, newidx, aliases) == expected class TestGetTierPreference(TestCase): """TestGetTierPreference @@ -348,28 +328,28 @@ def test_get_tier_preference1(self): client = Mock() roles = ['data_cold', 'data_frozen', 'data_hot', 'data_warm'] client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} - assert get_tier_preference(client) == 'data_frozen' + assert getters.get_tier_preference(client) == 'data_frozen' def test_get_tier_preference2(self): """test_get_tier_preference2""" client = Mock() roles = ['data_cold', 'data_hot', 'data_warm'] client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} - assert get_tier_preference(client) == 'data_cold,data_warm,data_hot' + assert getters.get_tier_preference(client) == 'data_cold,data_warm,data_hot' def test_get_tier_preference3(self): """test_get_tier_preference3""" client = Mock() roles = ['data_content'] client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} - assert get_tier_preference(client) == 'data_content' + assert getters.get_tier_preference(client) == 'data_content' def test_get_tier_preference4(self): """test_get_tier_preference4""" client = Mock() roles = ['data_cold', 'data_frozen', 'data_hot', 'data_warm'] client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} - assert get_tier_preference(client, target_tier='data_cold') == 'data_cold,data_warm,data_hot' + assert getters.get_tier_preference(client, target_tier='data_cold') == 'data_cold,data_warm,data_hot' def test_get_tier_preference5(self): """test_get_tier_preference5""" client = Mock() roles = ['data_content'] client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} - assert get_tier_preference(client, target_tier='data_hot') == 'data_content' \ No newline at end of file + assert getters.get_tier_preference(client, target_tier='data_hot') == 'data_content'