Skip to content

Commit

Permalink
Merge branch '8.x' into 8.0
Browse files Browse the repository at this point in the history
  • Loading branch information
untergeek committed Apr 3, 2024
2 parents 41c6f49 + e5d4460 commit d5a1a8f
Show file tree
Hide file tree
Showing 31 changed files with 291 additions and 137 deletions.
2 changes: 1 addition & 1 deletion curator/_version.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
"""Curator Version"""
__version__ = '8.0.13'
__version__ = '8.0.14'
4 changes: 2 additions & 2 deletions curator/actions/alias.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def add(self, ilo, warn_if_no_indices=False):
:type ilo: :py:class:`~.curator.indexlist.IndexList`
"""
verify_index_list(ilo)
self.loggit.debug('ADD -> ILO = %s', ilo)
self.loggit.debug('ADD -> ILO = %s', ilo.indices)
if not self.client:
self.client = ilo.client
self.name = parse_datemath(self.client, self.name)
Expand Down Expand Up @@ -80,7 +80,7 @@ def remove(self, ilo, warn_if_no_indices=False):
:type ilo: :py:class:`~.curator.indexlist.IndexList`
"""
verify_index_list(ilo)
self.loggit.debug('REMOVE -> ILO = %s', ilo)
self.loggit.debug('REMOVE -> ILO = %s', ilo.indices)
if not self.client:
self.client = ilo.client
self.name = parse_datemath(self.client, self.name)
Expand Down
16 changes: 10 additions & 6 deletions curator/actions/cold2frozen.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Snapshot and Restore action classes"""
import logging
from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference
from curator.helpers.getters import get_alias_actions, get_tier_preference, meta_getter
from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list
from curator.helpers.utils import report_failure
from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException
Expand Down Expand Up @@ -76,7 +76,8 @@ def action_generator(self):
:rtype: dict
"""
for idx in self.index_list.indices:
idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index']
idx_settings = meta_getter(self.client, idx, get='settings')
self.loggit.debug('Index %s has settings: %s', idx, idx_settings)
if has_lifecycle_name(idx_settings):
self.loggit.critical(
'Index %s is associated with an ILM policy and this action will never work on '
Expand All @@ -85,13 +86,16 @@ def action_generator(self):
if is_idx_partial(idx_settings):
self.loggit.critical('Index %s is already in the frozen tier', idx)
raise SearchableSnapshotException('Index is already in frozen tier')

snap = idx_settings['store']['snapshot']['snapshot_name']
snap_idx = idx_settings['store']['snapshot']['index_name']
repo = idx_settings['store']['snapshot']['repository_name']
aliases = self.client.indices.get(index=idx)[idx]['aliases']
msg = f'Index {idx} Snapshot name: {snap}, Snapshot index: {snap_idx}, repo: {repo}'
self.loggit.debug(msg)

aliases = meta_getter(self.client, idx, get='alias')

prefix = get_frozen_prefix(snap_idx, idx)
renamed = f'{prefix}{snap_idx}'
renamed = f'partial-{idx}'

if not self.index_settings:
self.index_settings = {
Expand Down Expand Up @@ -187,7 +191,7 @@ def cleanup(self, current_idx, newidx):
def do_action(self):
"""
Do the actions outlined:
Extract values from generated kwargs
Mount
Verify
Update Aliases
Expand Down
11 changes: 10 additions & 1 deletion curator/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,22 @@ def process_action(client, action_def, dry_run=False):
logger = logging.getLogger(__name__)
logger.debug('Configuration dictionary: %s', action_def.action_dict)
mykwargs = {}
search_pattern = '_all'

logger.critical('INITIAL Action kwargs: %s', mykwargs)
# Add some settings to mykwargs...
if action_def.action == 'delete_indices':
mykwargs['master_timeout'] = 30

### Update the defaults with whatever came with opts, minus any Nones
mykwargs.update(prune_nones(action_def.options))

# Pop out the search_pattern option, if present.
if 'search_pattern' in mykwargs:
search_pattern = mykwargs.pop('search_pattern')

logger.debug('Action kwargs: %s', mykwargs)
logger.critical('Post search_pattern Action kwargs: %s', mykwargs)

### Set up the action ###
logger.debug('Running "%s"', action_def.action.upper())
Expand All @@ -120,8 +128,9 @@ def process_action(client, action_def, dry_run=False):
mykwargs.pop('repository') # We don't need to send this value to the action
action_def.instantiate('list_obj', client, repository=action_def.options['repository'])
else:
action_def.instantiate('list_obj', client)
action_def.instantiate('list_obj', client, search_pattern=search_pattern)
action_def.list_obj.iterate_filters({'filters': action_def.filters})
logger.critical('Pre Instantiation Action kwargs: %s', mykwargs)
action_def.instantiate('action_cls', action_def.list_obj, **mykwargs)
### Do the action
if dry_run:
Expand Down
3 changes: 3 additions & 0 deletions curator/cli_singletons/allocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from curator.cli_singletons.utils import validate_filter_json

@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--key', type=str, required=True, help='Node identification tag')
@click.option('--value', type=str, default=None, help='Value associated with --key')
@click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude']))
Expand Down Expand Up @@ -47,6 +48,7 @@
@click.pass_context
def allocation(
ctx,
search_pattern,
key,
value,
allocation_type,
Expand All @@ -61,6 +63,7 @@ def allocation(
Shard Routing Allocation
"""
manual_options = {
'search_pattern': search_pattern,
'key': key,
'value': value,
'allocation_type': allocation_type,
Expand Down
6 changes: 5 additions & 1 deletion curator/cli_singletons/close.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from curator.cli_singletons.utils import validate_filter_json

@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed')
@click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed')
@click.option(
Expand All @@ -24,11 +25,14 @@
required=True
)
@click.pass_context
def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, filter_list):
def close(
ctx, search_pattern, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices,
filter_list):
"""
Close Indices
"""
manual_options = {
'search_pattern': search_pattern,
'skip_flush': skip_flush,
'delete_aliases': delete_aliases,
'allow_ilm_indices': allow_ilm_indices,
Expand Down
5 changes: 3 additions & 2 deletions curator/cli_singletons/delete.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#### Indices ####
@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option(
'--ignore_empty_list',
is_flag=True,
Expand All @@ -23,15 +24,15 @@
required=True
)
@click.pass_context
def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
def delete_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list):
"""
Delete Indices
"""
# ctx.info_name is the name of the function or name specified in @click.command decorator
action = CLIAction(
'delete_indices',
ctx.obj['configdict'],
{'allow_ilm_indices':allow_ilm_indices},
{'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices},
filter_list,
ignore_empty_list
)
Expand Down
6 changes: 5 additions & 1 deletion curator/cli_singletons/forcemerge.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from curator.cli_singletons.utils import validate_filter_json

@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option(
'--max_num_segments',
type=int,
Expand Down Expand Up @@ -32,11 +33,14 @@
help='JSON array of filters selecting indices to act on.',
required=True)
@click.pass_context
def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list):
def forcemerge(
ctx, search_pattern, max_num_segments, delay, ignore_empty_list, allow_ilm_indices,
filter_list):
"""
forceMerge Indices (reduce segment count)
"""
manual_options = {
'search_pattern': search_pattern,
'max_num_segments': max_num_segments,
'delay': delay,
'allow_ilm_indices': allow_ilm_indices,
Expand Down
6 changes: 6 additions & 0 deletions curator/cli_singletons/object_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ def __init__(self, action, configdict, option_dict, filter_list, ignore_empty_li
self.check_options(option_dict)
else:
self.options = option_dict

# Pop out search_pattern if it's there
self.search_pattern = '_all'
if 'search_pattern' in self.options:
self.search_pattern = self.options.pop('search_pattern')

# Extract allow_ilm_indices so it can be handled separately.
if 'allow_ilm_indices' in self.options:
self.allow_ilm = self.options.pop('allow_ilm_indices')
Expand Down
5 changes: 3 additions & 2 deletions curator/cli_singletons/open_indices.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from curator.cli_singletons.utils import validate_filter_json

@click.command(name='open')
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option(
'--ignore_empty_list',
is_flag=True,
Expand All @@ -22,15 +23,15 @@
required=True
)
@click.pass_context
def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
def open_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list):
"""
Open Indices
"""
# ctx.info_name is the name of the function or name specified in @click.command decorator
action = CLIAction(
ctx.info_name,
ctx.obj['configdict'],
{'allow_ilm_indices':allow_ilm_indices},
{'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices},
filter_list,
ignore_empty_list
)
Expand Down
6 changes: 5 additions & 1 deletion curator/cli_singletons/replicas.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from curator.cli_singletons.utils import validate_filter_json

@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--count', type=int, required=True, help='Number of replicas (max 10)')
@click.option(
'--wait_for_completion/--no-wait_for_completion',
Expand All @@ -29,11 +30,14 @@
required=True
)
@click.pass_context
def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list):
def replicas(
ctx, search_pattern, count, wait_for_completion, ignore_empty_list, allow_ilm_indices,
filter_list):
"""
Change Replica Count
"""
manual_options = {
'search_pattern': search_pattern,
'count': count,
'wait_for_completion': wait_for_completion,
'allow_ilm_indices': allow_ilm_indices,
Expand Down
7 changes: 5 additions & 2 deletions curator/cli_singletons/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,25 @@

# pylint: disable=line-too-long
@click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots'))
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True)
@click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True)
@click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True)
@click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
@click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.')
@click.pass_context
def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, filter_list):
def show_indices(
ctx, search_pattern, verbose, header, epoch, ignore_empty_list, allow_ilm_indices,
filter_list):
"""
Show Indices
"""
# ctx.info_name is the name of the function or name specified in @click.command decorator
action = CLIAction(
'show_indices',
ctx.obj['configdict'],
{'allow_ilm_indices': allow_ilm_indices},
{'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices},
filter_list,
ignore_empty_list
)
Expand Down
6 changes: 4 additions & 2 deletions curator/cli_singletons/shrink.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

# pylint: disable=line-too-long
@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True)
@click.option('--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict)
@click.option('--number_of_shards', default=1, type=int, help='Shrink to this many shards per index')
Expand All @@ -25,15 +26,16 @@
@click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
@click.pass_context
def shrink(
ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix,
shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings,
ctx, search_pattern, shrink_node, node_filters, number_of_shards, number_of_replicas,
shrink_prefix, shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings,
wait_for_active_shards, wait_for_rebalance, wait_for_completion, wait_interval, max_wait,
ignore_empty_list, allow_ilm_indices, filter_list
):
"""
Shrink Indices to --number_of_shards
"""
manual_options = {
'search_pattern': search_pattern,
'shrink_node': shrink_node,
'node_filters': node_filters,
'number_of_shards': number_of_shards,
Expand Down
4 changes: 3 additions & 1 deletion curator/cli_singletons/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

# pylint: disable=line-too-long
@click.command()
@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
@click.option('--repository', type=str, required=True, help='Snapshot repository')
@click.option('--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S')
@click.option('--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.')
Expand All @@ -19,14 +20,15 @@
@click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
@click.pass_context
def snapshot(
ctx, repository, name, ignore_unavailable, include_global_state, partial,
ctx, search_pattern, repository, name, ignore_unavailable, include_global_state, partial,
skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list,
allow_ilm_indices, filter_list
):
"""
Snapshot Indices
"""
manual_options = {
'search_pattern': search_pattern,
'name': name,
'repository': repository,
'ignore_unavailable': ignore_unavailable,
Expand Down
6 changes: 6 additions & 0 deletions curator/defaults/option_defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,12 @@ def cluster_routing_value():
"""
return {Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')}

def search_pattern():
"""
:returns: ``{Optional('search_pattern', default='_all'): Any(*string_types)}``
"""
return {Optional('search_pattern', default='_all'): Any(*string_types)}

def shrink_node():
"""
:returns: ``{Required('shrink_node'): Any(*string_types)}``
Expand Down
Loading

0 comments on commit d5a1a8f

Please sign in to comment.