From dd762e6c5f185d50a9f99aef2432072c908c6289 Mon Sep 17 00:00:00 2001 From: Nicolas Boutet Date: Fri, 5 Jul 2024 18:34:34 +0200 Subject: [PATCH 1/2] Fix wrong log level for several debug messages (#1714) --- curator/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/curator/cli.py b/curator/cli.py index 1a2f923b..719f1432 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -88,7 +88,7 @@ def process_action(client, action_def, dry_run=False): mykwargs = {} search_pattern = '_all' - logger.critical('INITIAL Action kwargs: %s', mykwargs) + logger.debug('INITIAL Action kwargs: %s', mykwargs) # Add some settings to mykwargs... if action_def.action == 'delete_indices': mykwargs['master_timeout'] = 30 @@ -101,7 +101,7 @@ def process_action(client, action_def, dry_run=False): search_pattern = mykwargs.pop('search_pattern') logger.debug('Action kwargs: %s', mykwargs) - logger.critical('Post search_pattern Action kwargs: %s', mykwargs) + logger.debug('Post search_pattern Action kwargs: %s', mykwargs) ### Set up the action ### logger.debug('Running "%s"', action_def.action.upper()) @@ -130,7 +130,7 @@ def process_action(client, action_def, dry_run=False): else: action_def.instantiate('list_obj', client, search_pattern=search_pattern) action_def.list_obj.iterate_filters({'filters': action_def.filters}) - logger.critical('Pre Instantiation Action kwargs: %s', mykwargs) + logger.debug('Pre Instantiation Action kwargs: %s', mykwargs) action_def.instantiate('action_cls', action_def.list_obj, **mykwargs) ### Do the action if dry_run: From d2d09eca1642cdcfdc62c32b3b320826fefe0879 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Tue, 6 Aug 2024 19:08:36 -0600 Subject: [PATCH 2/2] Release prep for 8.0.16 (#1716) * Commit local changes to bring in updates from master * Release prep for 8.0.16 Yeah, I know. The branch name was for something else. But a request came in and I didn't want to make a new branch. --- .gitignore | 195 +++++++++-- curator/_version.py | 3 +- curator/actions/reindex.py | 167 +++++---- curator/classdef.py | 78 +++-- curator/cli.py | 160 ++++++--- curator/cli_singletons/delete.py | 55 ++- curator/defaults/filter_elements.py | 236 ++++++++----- curator/defaults/option_defaults.py | 518 ++++++++++++++++++++-------- curator/defaults/settings.py | 107 +++--- curator/helpers/date_ops.py | 256 ++++++++------ curator/helpers/testers.py | 94 +++-- curator/singletons.py | 69 +++- curator/validators/actions.py | 35 +- docker_test/.env | 2 +- docs/Changelog.rst | 25 +- docs/asciidoc/index.asciidoc | 8 +- docs/conf.py | 6 +- docs/helpers.rst | 2 - mypy.ini | 2 + pyproject.toml | 19 +- pytest.ini | 2 + tests/integration/test_reindex.py | 212 +++++++----- tests/unit/test_helpers_waiters.py | 225 +++++++++--- 23 files changed, 1728 insertions(+), 748 deletions(-) create mode 100644 mypy.ini create mode 100644 pytest.ini diff --git a/.gitignore b/.gitignore index f09ccdfc..d8023d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,31 +1,178 @@ -*.egg +.DS_Store localhost.es oneliners.py -*.py[co] -*.zip -*~ -.*.swp -.DS_Store -.coverage -.eggs -.idea -.vscode -build cacert.pem -cover -cov_html/ -coverage.xml -dist -docs/_build -docs/asciidoc/html_docs/ +docs/ docker_test/repo/ docker_test/curatortestenv docker_test/scripts/Dockerfile -elasticsearch_curator.egg-info -elasticsearch_curator_dev.egg-info html_docs/ -index.html -pytest.ini -samples -scratch -test/.DS_Store + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +.black +.flake8 +pylintrc +pylintrc.toml + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.mypy.ini +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.vscode diff --git a/curator/_version.py b/curator/_version.py index d776fb9f..46f6a51d 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,3 @@ """Curator Version""" -__version__ = '8.0.15' + +__version__ = '8.0.16' diff --git a/curator/actions/reindex.py b/curator/actions/reindex.py index 5c799699..3fc4802d 100644 --- a/curator/actions/reindex.py +++ b/curator/actions/reindex.py @@ -1,42 +1,65 @@ """Reindex action class""" + import logging from copy import deepcopy -# pylint: disable=broad-except -from es_client.builder import ClientArgs, OtherArgs, Builder +from dotmap import DotMap # type: ignore + +# pylint: disable=broad-except, R0902,R0912,R0913,R0914,R0915 +from es_client.builder import Builder from es_client.helpers.utils import ensure_list, verify_url_schema from es_client.exceptions import ConfigurationError from curator.exceptions import CuratorException, FailedExecution, NoIndices -from curator.exceptions import ConfigurationError as CuratorConfigError # Separate from es_client + +# Separate from es_client +from curator.exceptions import ConfigurationError as CuratorConfigError from curator.helpers.testers import verify_index_list from curator.helpers.utils import report_failure from curator.helpers.waiters import wait_for_it from curator import IndexList + class Reindex: """Reindex Action Class""" + def __init__( - self, ilo, request_body, refresh=True, requests_per_second=-1, slices=1, timeout=60, - wait_for_active_shards=1, wait_for_completion=True, max_wait=-1, wait_interval=9, - remote_certificate=None, remote_client_cert=None, remote_client_key=None, - remote_filters=None, migration_prefix='', migration_suffix='' + self, + ilo, + request_body, + refresh=True, + requests_per_second=-1, + slices=1, + timeout=60, + wait_for_active_shards=1, + wait_for_completion=True, + max_wait=-1, + wait_interval=9, + remote_certificate=None, + remote_client_cert=None, + remote_client_key=None, + remote_filters=None, + migration_prefix='', + migration_suffix='', ): """ :param ilo: An IndexList Object - :param request_body: The body to send to :py:meth:`~.elasticsearch.Elasticsearch.reindex`, - which must be complete and usable, as Curator will do no vetting of the request_body. + :param request_body: The body to send to + :py:meth:`~.elasticsearch.Elasticsearch.reindex`, which must be + complete and usable, as Curator will do no vetting of the request_body. If it fails to function, Curator will return an exception. - :param refresh: Whether to refresh the entire target index after the operation is complete. - :param requests_per_second: The throttle to set on this request in sub-requests per second. - ``-1`` means set no throttle as does ``unlimited`` which is the only non-float this - accepts. - :param slices: The number of slices this task should be divided into. ``1`` means the task - will not be sliced into subtasks. (Default: ``1``) - :param timeout: The length in seconds each individual bulk request should wait for shards - that are unavailable. (default: ``60``) - :param wait_for_active_shards: Sets the number of shard copies that must be active before - proceeding with the reindex operation. (Default: ``1``) means the primary shard only. - Set to ``all`` for all shard copies, otherwise set to any non-negative value less than - or equal to the total number of copies for the shard (number of replicas + 1) + :param refresh: Whether to refresh the entire target index after the + operation is complete. + :param requests_per_second: The throttle to set on this request in + sub-requests per second. ``-1`` means set no throttle as does + ``unlimited`` which is the only non-float this accepts. + :param slices: The number of slices this task should be divided into. + ``1`` means the task will not be sliced into subtasks. (Default: ``1``) + :param timeout: The length in seconds each individual bulk request should + wait for shards that are unavailable. (default: ``60``) + :param wait_for_active_shards: Sets the number of shard copies that must be + active before proceeding with the reindex operation. (Default: ``1``) + means the primary shard only. Set to ``all`` for all shard copies, + otherwise set to any non-negative value less than or equal to the total + number of copies for the shard (number of replicas + 1) :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` @@ -71,7 +94,8 @@ def __init__( #: Object attribute that gets the value of param ``request_body``. self.body = request_body self.loggit.debug('REQUEST_BODY = %s', request_body) - #: The :py:class:`~.curator.indexlist.IndexList` object passed from param ``ilo`` + #: The :py:class:`~.curator.indexlist.IndexList` object passed from + #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` @@ -82,8 +106,8 @@ def __init__( self.requests_per_second = requests_per_second #: Object attribute that gets the value of param ``slices``. self.slices = slices - #: Object attribute that gets the value of param ``timeout``, convert to :py:class:`str` - #: and add ``s`` for seconds. + #: Object attribute that gets the value of param ``timeout``, convert to + #: :py:class:`str` and add ``s`` for seconds. self.timeout = f'{timeout}s' #: Object attribute that gets the value of param ``wait_for_active_shards``. self.wait_for_active_shards = wait_for_active_shards @@ -120,19 +144,20 @@ def __init__( # REINDEX_SELECTION is the designated token. If you use this for the # source "index," it will be replaced with the list of indices from the # provided 'ilo' (index list object). - if self.body['source']['index'] == 'REINDEX_SELECTION' \ - and not self.remote: + if self.body['source']['index'] == 'REINDEX_SELECTION' and not self.remote: self.body['source']['index'] = self.index_list.indices # Remote section elif self.remote: - rclient_args = ClientArgs() - rother_args = OtherArgs() + rclient_args = DotMap() + rother_args = DotMap() self.loggit.debug('Remote reindex request detected') if 'host' not in self.body['source']['remote']: raise CuratorConfigError('Missing remote "host"') try: - rclient_args.hosts = verify_url_schema(self.body['source']['remote']['host']) + rclient_args.hosts = verify_url_schema( + self.body['source']['remote']['host'] + ) except ConfigurationError as exc: raise CuratorConfigError(exc) from exc @@ -154,7 +179,7 @@ def __init__( # Let's set a decent remote timeout for initially reading # the indices on the other side, and collecting their metadata - rclient_args.remote_timeout = 180 + rclient_args.request_timeout = 180 # The rest only applies if using filters for remote indices if self.body['source']['index'] == 'REINDEX_SELECTION': @@ -167,18 +192,19 @@ def __init__( f'certificate={remote_certificate} ' f'client_cert={remote_client_cert} ' f'client_key={remote_client_key} ' - f'request_timeout={rclient_args.remote_timeout} ' + f'request_timeout={rclient_args.request_timeout} ' f'skip_version_test=True' ) self.loggit.debug(msg) remote_config = { 'elasticsearch': { - 'client': rclient_args.asdict(), - 'other_settings': rother_args.asdict() + 'client': rclient_args.toDict(), + 'other_settings': rother_args.toDict(), } } - try: # let's try to build a remote connection with these! - builder = Builder(configdict=remote_config, version_min=(1,0,0)) + try: # let's try to build a remote connection with these! + builder = Builder(configdict=remote_config) + builder.version_min = (1, 0, 0) builder.connect() rclient = builder.client except Exception as err: @@ -194,7 +220,8 @@ def __init__( rio.empty_list_check() except NoIndices as exc: raise FailedExecution( - 'No actionable remote indices selected after applying filters.' + 'No actionable remote indices selected after applying ' + 'filters.' ) from exc self.body['source']['index'] = rio.indices except Exception as err: @@ -214,34 +241,46 @@ def _get_reindex_args(self, source, dest): # thing if wait_for_completion is set to True. Report the task_id # either way. reindex_args = { - 'refresh':self.refresh, + 'refresh': self.refresh, 'requests_per_second': self.requests_per_second, 'slices': self.slices, 'timeout': self.timeout, 'wait_for_active_shards': self.wait_for_active_shards, 'wait_for_completion': False, } - for keyname in ['dest', 'source', 'conflicts', 'max_docs', 'size', '_source', 'script']: + for keyname in [ + 'dest', + 'source', + 'conflicts', + 'max_docs', + 'size', + '_source', + 'script', + ]: if keyname in self.body: reindex_args[keyname] = self.body[keyname] - # Mimic the _get_request_body(source, dest) behavior by casting these values here instead + # Mimic the _get_request_body(source, dest) behavior by casting these values + # here instead reindex_args['dest']['index'] = dest reindex_args['source']['index'] = source return reindex_args def get_processed_items(self, task_id): """ - This function calls :py:func:`~.elasticsearch.client.TasksClient.get` with the provided - ``task_id``. It will get the value from ``'response.total'`` as the total number of - elements processed during reindexing. If the value is not found, it will return ``-1`` + This function calls :py:func:`~.elasticsearch.client.TasksClient.get` with + the provided ``task_id``. It will get the value from ``'response.total'`` + as the total number of elements processed during reindexing. If the value is + not found, it will return ``-1`` - :param task_id: A task_id which ostensibly matches a task searchable in the tasks API. + :param task_id: A task_id which ostensibly matches a task searchable in the + tasks API. """ try: task_data = self.client.tasks.get(task_id=task_id) except Exception as exc: raise CuratorException( - f'Unable to obtain task information for task_id "{task_id}". Exception {exc}' + f'Unable to obtain task information for task_id "{task_id}". ' + f'Exception {exc}' ) from exc total_processed_items = -1 task = task_data['task'] @@ -260,7 +299,10 @@ def _post_run_quick_check(self, index_name, task_id): # if no documents processed, the target index "dest" won't exist processed_items = self.get_processed_items(task_id) if processed_items == 0: - msg = f'No items were processed. Will not check if target index "{index_name}" exists' + msg = ( + f'No items were processed. Will not check if target index ' + f'"{index_name}" exists' + ) self.loggit.info(msg) else: # Verify the destination index is there after the fact @@ -269,20 +311,20 @@ def _post_run_quick_check(self, index_name, task_id): if not index_exists and not alias_instead: # pylint: disable=logging-fstring-interpolation self.loggit.error( - f'The index described as "{index_name}" was not found after the reindex ' - f'operation. Check Elasticsearch logs for more ' + f'The index described as "{index_name}" was not found after the ' + f'reindex operation. Check Elasticsearch logs for more ' f'information.' ) if self.remote: # pylint: disable=logging-fstring-interpolation self.loggit.error( f'Did you forget to add "reindex.remote.whitelist: ' - f'{self.remote_host}:{self.remote_port}" to the elasticsearch.yml file on ' - f'the "dest" node?' + f'{self.remote_host}:{self.remote_port}" to the ' + f'elasticsearch.yml file on the "dest" node?' ) raise FailedExecution( - f'Reindex failed. The index or alias identified by "{index_name}" was ' - f'not found.' + f'Reindex failed. The index or alias identified by "{index_name}" ' + f'was not found.' ) def sources(self): @@ -290,7 +332,7 @@ def sources(self): dest = self.body['dest']['index'] source_list = ensure_list(self.body['source']['index']) self.loggit.debug('source_list: %s', source_list) - if not source_list or source_list == ['REINDEX_SELECTED']: # Empty list + if not source_list or source_list == ['REINDEX_SELECTED']: # Empty list raise NoIndices if not self.migration: yield self.body['source']['index'], dest @@ -323,9 +365,9 @@ def do_dry_run(self): def do_action(self): """ Execute :py:meth:`~.elasticsearch.Elasticsearch.reindex` operation with the - ``request_body`` from :py:meth:`_get_request_body` and arguments :py:attr:`refresh`, - :py:attr:`requests_per_second`, :py:attr:`slices`, :py:attr:`timeout`, - :py:attr:`wait_for_active_shards`, and :py:attr:`wfc`. + ``request_body`` from :py:meth:`_get_request_body` and arguments + :py:attr:`refresh`, :py:attr:`requests_per_second`, :py:attr:`slices`, + :py:attr:`timeout`, :py:attr:`wait_for_active_shards`, and :py:attr:`wfc`. """ try: # Loop over all sources (default will only be one) @@ -337,20 +379,25 @@ def do_action(self): self.loggit.debug('TASK ID = %s', response['task']) if self.wfc: wait_for_it( - self.client, 'reindex', task_id=response['task'], - wait_interval=self.wait_interval, max_wait=self.max_wait + self.client, + 'reindex', + task_id=response['task'], + wait_interval=self.wait_interval, + max_wait=self.max_wait, ) self._post_run_quick_check(dest, response['task']) else: msg = ( - f'"wait_for_completion" set to {self.wfc}. Remember to check task_id ' - f"\"{response['task']}\" for successful completion manually." + f'"wait_for_completion" set to {self.wfc}. Remember to check ' + f"task_id \"{response['task']}\" for successful completion " + f"manually." ) self.loggit.warning(msg) except NoIndices as exc: raise NoIndices( - 'Source index must be list of actual indices. It must not be an empty list.' + 'Source index must be list of actual indices. It must not be an empty ' + 'list.' ) from exc except Exception as exc: report_failure(exc) diff --git a/curator/classdef.py b/curator/classdef.py index be91de75..a4553d7d 100644 --- a/curator/classdef.py +++ b/curator/classdef.py @@ -1,4 +1,5 @@ """Other Classes""" + import logging from es_client.exceptions import FailedValidation from es_client.helpers.schemacheck import password_filter @@ -8,15 +9,17 @@ from curator.exceptions import ConfigurationError from curator.helpers.testers import validate_actions -# Let me tell you the story of the nearly wasted afternoon and the research that went into this -# seemingly simple work-around. Actually, no. It's even more wasted time writing that story here. -# Suffice to say that I couldn't use the CLASS_MAP with class objects to directly map them to class -# instances. The Wrapper class and the ActionDef.instantiate method do all of the work for me, -# allowing me to easily and cleanly pass *args and **kwargs to the individual action classes of -# CLASS_MAP. +# Let me tell you the story of the nearly wasted afternoon and the research that went +# into this seemingly simple work-around. Actually, no. It's even more wasted time +# writing that story here. Suffice to say that I couldn't use the CLASS_MAP with class +# objects to directly map them to class instances. The Wrapper class and the +# ActionDef.instantiate method do all of the work for me, allowing me to easily and +# cleanly pass *args and **kwargs to the individual action classes of CLASS_MAP. + class Wrapper: """Wrapper Class""" + def __init__(self, cls): """Instantiate with passed Class (not instance or object) @@ -32,23 +35,25 @@ def set_instance(self, *args, **kwargs): self.class_instance = self.class_object(*args, **kwargs) def get_instance(self, *args, **kwargs): - """Return the instance with ``*args`` and ``**kwargs`` - """ + """Return the instance with ``*args`` and ``**kwargs``""" self.set_instance(*args, **kwargs) return self.class_instance + class ActionsFile: """Class to parse and verify entire actions file Individual actions are :py:class:`~.curator.classdef.ActionDef` objects """ + def __init__(self, action_file): self.logger = logging.getLogger(__name__) #: The full, validated configuration from ``action_file``. self.fullconfig = self.get_validated(action_file) self.logger.debug('Action Configuration: %s', password_filter(self.fullconfig)) - #: A dict of all actions in the provided configuration. Each original key name is preserved - #: and the value is now an :py:class:`~.curator.classdef.ActionDef`, rather than a dict. + #: A dict of all actions in the provided configuration. Each original key name + #: is preserved and the value is now an + #: :py:class:`~.curator.classdef.ActionDef`, rather than a dict. self.actions = None self.set_actions(self.fullconfig['actions']) @@ -69,8 +74,9 @@ def get_validated(self, action_file): def parse_actions(self, all_actions): """Parse the individual actions found in ``all_actions['actions']`` - :param all_actions: All actions, each its own dictionary behind a numeric key. Making the - keys numeric guarantees that if they are sorted, they will always be executed in order. + :param all_actions: All actions, each its own dictionary behind a numeric key. + Making the keys numeric guarantees that if they are sorted, they will + always be executed in order. :type all_actions: dict @@ -85,13 +91,15 @@ def parse_actions(self, all_actions): def set_actions(self, all_actions): """Set the actions via :py:meth:`~.curator.classdef.ActionsFile.parse_actions` - :param all_actions: All actions, each its own dictionary behind a numeric key. Making the - keys numeric guarantees that if they are sorted, they will always be executed in order. + :param all_actions: All actions, each its own dictionary behind a numeric key. + Making the keys numeric guarantees that if they are sorted, they will + always be executed in order. :type all_actions: dict :rtype: None """ self.actions = self.parse_actions(all_actions) + # In this case, I just don't care that pylint thinks I'm overdoing it with attributes # pylint: disable=too-many-instance-attributes class ActionDef: @@ -99,6 +107,7 @@ class ActionDef: Instances of this class represent an individual action from an action file. """ + def __init__(self, action_dict): #: The whole action dictionary self.action_dict = action_dict @@ -111,7 +120,8 @@ def __init__(self, action_dict): #: Only when action is alias will this be a :py:class:`~.curator.IndexList` self.alias_removes = None #: The list class, either :py:class:`~.curator.IndexList` or - #: :py:class:`~.curator.SnapshotList`. Default is :py:class:`~.curator.IndexList` + #: :py:class:`~.curator.SnapshotList`. Default is + #: :py:class:`~.curator.IndexList` self.list_obj = Wrapper(IndexList) #: The action ``description`` self.description = None @@ -136,32 +146,38 @@ def __init__(self, action_dict): def instantiate(self, attribute, *args, **kwargs): """ - Convert ``attribute`` from being a :py:class:`~.curator.classdef.Wrapper` of a Class to an - instantiated object of that Class. + Convert ``attribute`` from being a :py:class:`~.curator.classdef.Wrapper` of a + Class to an instantiated object of that Class. This is madness or genius. You decide. This entire method plus the - :py:class:`~.curator.classdef.Wrapper` class came about because I couldn't cleanly - instantiate a class variable into a class object. It works, and that's good enough for me. + :py:class:`~.curator.classdef.Wrapper` class came about because I couldn't + cleanly instantiate a class variable into a class object. It works, and that's + good enough for me. - :param attribute: The `name` of an attribute that references a Wrapper class instance + :param attribute: The `name` of an attribute that references a Wrapper class + instance :type attribute: str """ try: wrapper = getattr(self, attribute) except AttributeError as exc: - raise AttributeError(f'Bad Attribute: {attribute}. Exception: {exc}') from exc + raise AttributeError( + f'Bad Attribute: {attribute}. Exception: {exc}' + ) from exc setattr(self, attribute, self.get_obj_instance(wrapper, *args, **kwargs)) def get_obj_instance(self, wrapper, *args, **kwargs): """Get the class instance wrapper identified by ``wrapper`` - Pass all other args and kwargs to the :py:meth:`~.curator.classdef.Wrapper.get_instance` - method. + Pass all other args and kwargs to the + :py:meth:`~.curator.classdef.Wrapper.get_instance` method. - :returns: An instance of the class that :py:class:`~.curator.classdef.Wrapper` is wrapping + :returns: An instance of the class that :py:class:`~.curator.classdef.Wrapper` + is wrapping """ if not isinstance(wrapper, Wrapper): raise ConfigurationError( - f'{__name__} was passed wrapper which was of type {type(wrapper)}') + f'{__name__} was passed wrapper which was of type {type(wrapper)}' + ) return wrapper.get_instance(*args, **kwargs) def set_alias_extras(self): @@ -175,7 +191,8 @@ def get_action_class(self): Do extra setup when action is ``alias`` Set :py:attr:`list_obj` to :py:class:`~.curator.SnapshotList` when - :py:attr:`~.curator.classdef.ActionDef.action` is ``delete_snapshots`` or ``restore`` + :py:attr:`~.curator.classdef.ActionDef.action` is ``delete_snapshots`` or + ``restore`` """ self.action_cls = Wrapper(CLASS_MAP[self.action]) @@ -186,15 +203,15 @@ def get_action_class(self): def set_option_attrs(self): """ - Iteratively get the keys and values from :py:attr:`~.curator.classdef.ActionDef.options` - and set the attributes + Iteratively get the keys and values from + :py:attr:`~.curator.classdef.ActionDef.options` and set the attributes """ attmap = { 'disable_action': 'disabled', 'continue_if_exception': 'cif', 'ignore_empty_list': 'iel', 'allow_ilm_indices': 'allow_ilm', - 'timeout_override' : 'timeout_override' + 'timeout_override': 'timeout_override', } for key in self.action_dict['options']: if key in attmap: @@ -219,7 +236,8 @@ def log_the_options(self): logger = logging.getLogger('curator.cli.ActionDef') msg = ( f'For action {self.action}: disable_action={self.disabled}' - f'continue_if_exception={self.cif}, timeout_override={self.timeout_override}' + f'continue_if_exception={self.cif}, ' + f'timeout_override={self.timeout_override}, ' f'ignore_empty_list={self.iel}, allow_ilm_indices={self.allow_ilm}' ) logger.debug(msg) diff --git a/curator/cli.py b/curator/cli.py index 719f1432..ab348782 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -1,15 +1,27 @@ """Main CLI for Curator""" + import sys import logging import click from es_client.defaults import OPTION_DEFAULTS from es_client.helpers.config import ( - cli_opts, context_settings, generate_configdict, get_client, get_config, options_from_dict) + cli_opts, + context_settings, + generate_configdict, + get_client, + get_config, + options_from_dict, +) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.exceptions import ClientException from curator.classdef import ActionsFile -from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer, snapshot_actions +from curator.defaults.settings import ( + CLICK_DRYRUN, + default_config_file, + footer, + snapshot_actions, +) from curator.exceptions import NoIndices, NoSnapshots from curator.helpers.testers import ilm_policy_check from curator._version import __version__ @@ -17,25 +29,32 @@ ONOFF = {'on': '', 'off': 'no-'} click_opt_wrap = option_wrapper() +# pylint: disable=R0913, R0914, W0613, W0622, W0718 + + def ilm_action_skip(client, action_def): """ - Skip rollover action if ``allow_ilm_indices`` is ``false``. For all other non-snapshot actions, - add the ``ilm`` filtertype to the :py:attr:`~.curator.ActionDef.filters` list. + Skip rollover action if ``allow_ilm_indices`` is ``false``. For all other + non-snapshot actions, add the ``ilm`` filtertype to the + :py:attr:`~.curator.ActionDef.filters` list. :param action_def: An action object :type action_def: :py:class:`~.curator.classdef.ActionDef` - :returns: ``True`` if ``action_def.action`` is ``rollover`` and the alias identified by - ``action_def.options['name']`` is associated with an ILM policy. This hacky work-around is - because the Rollover action does not use :py:class:`~.curator.IndexList` + :returns: ``True`` if ``action_def.action`` is ``rollover`` and the alias + identified by ``action_def.options['name']`` is associated with an ILM + policy. This hacky work-around is because the Rollover action does not + use :py:class:`~.curator.IndexList` :rtype: bool """ logger = logging.getLogger(__name__) if not action_def.allow_ilm and action_def.action not in snapshot_actions(): if action_def.action == 'rollover': if ilm_policy_check(client, action_def.options['name']): - logger.info('Alias %s is associated with ILM policy.', action_def.options['name']) - # logger.info('Skipping action %s because allow_ilm_indices is false.', idx) + logger.info( + 'Alias %s is associated with ILM policy.', + action_def.options['name'], + ) return True elif action_def.filters: action_def.filters.append({'filtertype': 'ilm'}) @@ -43,6 +62,7 @@ def ilm_action_skip(client, action_def): action_def.filters = [{'filtertype': 'ilm'}] return False + def exception_handler(action_def, err): """Do the grunt work with the exception @@ -56,25 +76,35 @@ def exception_handler(action_def, err): if isinstance(err, (NoIndices, NoSnapshots)): if action_def.iel: logger.info( - 'Skipping action "%s" due to empty list: %s', action_def.action, type(err)) + 'Skipping action "%s" due to empty list: %s', + action_def.action, + type(err), + ) else: logger.error( 'Unable to complete action "%s". No actionable items in list: %s', - action_def.action, type(err)) + action_def.action, + type(err), + ) sys.exit(1) else: logger.error( - 'Failed to complete action: %s. %s: %s', action_def.action, type(err), err) + 'Failed to complete action: %s. %s: %s', action_def.action, type(err), err + ) if action_def.cif: logger.info( 'Continuing execution with next action because "continue_if_exception" ' - 'is set to True for action %s', action_def.action) + 'is set to True for action %s', + action_def.action, + ) else: sys.exit(1) + def process_action(client, action_def, dry_run=False): """ - Do the ``action`` in ``action_def.action``, using the associated options and any ``kwargs``. + Do the ``action`` in ``action_def.action``, using the associated options and + any ``kwargs``. :param client: A client connection object :param action_def: The ``action`` object @@ -93,7 +123,7 @@ def process_action(client, action_def, dry_run=False): if action_def.action == 'delete_indices': mykwargs['master_timeout'] = 30 - ### Update the defaults with whatever came with opts, minus any Nones + # Update the defaults with whatever came with opts, minus any Nones mykwargs.update(prune_nones(action_def.options)) # Pop out the search_pattern option, if present. @@ -103,7 +133,7 @@ def process_action(client, action_def, dry_run=False): logger.debug('Action kwargs: %s', mykwargs) logger.debug('Post search_pattern Action kwargs: %s', mykwargs) - ### Set up the action ### + # Set up the action logger.debug('Running "%s"', action_def.action.upper()) if action_def.action == 'alias': # Special behavior for this action, as it has 2 index lists @@ -115,33 +145,39 @@ def process_action(client, action_def, dry_run=False): action_def.alias_removes.iterate_filters(action_def.action_dict['remove']) action_def.action_cls.remove( action_def.alias_removes, - warn_if_no_indices=action_def.options['warn_if_no_indices']) + warn_if_no_indices=action_def.options['warn_if_no_indices'], + ) if 'add' in action_def.action_dict: logger.debug('Adding indices to alias "%s"', action_def.options['name']) action_def.alias_adds.iterate_filters(action_def.action_dict['add']) action_def.action_cls.add( - action_def.alias_adds, warn_if_no_indices=action_def.options['warn_if_no_indices']) + action_def.alias_adds, + warn_if_no_indices=action_def.options['warn_if_no_indices'], + ) elif action_def.action in ['cluster_routing', 'create_index', 'rollover']: action_def.instantiate('action_cls', client, **mykwargs) else: if action_def.action in ['delete_snapshots', 'restore']: - mykwargs.pop('repository') # We don't need to send this value to the action - action_def.instantiate('list_obj', client, repository=action_def.options['repository']) + mykwargs.pop('repository') # We don't need to send this value to the action + action_def.instantiate( + 'list_obj', client, repository=action_def.options['repository'] + ) else: action_def.instantiate('list_obj', client, search_pattern=search_pattern) action_def.list_obj.iterate_filters({'filters': action_def.filters}) logger.debug('Pre Instantiation Action kwargs: %s', mykwargs) action_def.instantiate('action_cls', action_def.list_obj, **mykwargs) - ### Do the action + # Do the action if dry_run: action_def.action_cls.do_dry_run() else: logger.debug('Doing the action here.') action_def.action_cls.do_action() + def run(ctx: click.Context) -> None: """ - :param ctx: The Click command context + :param ctx: The Click command context :type ctx: :py:class:`Context ` @@ -152,73 +188,105 @@ def run(ctx: click.Context) -> None: all_actions = ActionsFile(ctx.params['action_file']) for idx in sorted(list(all_actions.actions.keys())): action_def = all_actions.actions[idx] - ### Skip to next action if 'disabled' + # Skip to next action if 'disabled' if action_def.disabled: logger.info( 'Action ID: %s: "%s" not performed because "disable_action" ' - 'is set to True', idx, action_def.action + 'is set to True', + idx, + action_def.action, ) continue logger.info('Preparing Action ID: %s, "%s"', idx, action_def.action) # Override the timeout, if specified, otherwise use the default. if action_def.timeout_override: - ctx.obj['client_args'].request_timeout = action_def.timeout_override + ctx.obj['configdict']['elasticsearch']['client'][ + 'request_timeout' + ] = action_def.timeout_override # Create a client object for each action... logger.info('Creating client object and testing connection') try: - client = get_client(configdict={ - 'elasticsearch': { - 'client': prune_nones(ctx.obj['client_args'].asdict()), - 'other_settings': prune_nones(ctx.obj['other_args'].asdict()) - } - }) + client = get_client(configdict=ctx.obj['configdict']) except ClientException as exc: - # No matter where logging is set to go, make sure we dump these messages to the CLI + # No matter where logging is set to go, make sure we dump these messages to + # the CLI click.echo('Unable to establish client connection to Elasticsearch!') click.echo(f'Exception: {exc}') sys.exit(1) + except Exception as other: + logger.debug('Fatal exception encountered: %s', other) - ### Filter ILM indices unless expressly permitted + # Filter ILM indices unless expressly permitted if ilm_action_skip(client, action_def): continue - ########################## - ### Process the action ### - ########################## - msg = f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' + # + # Process the action + # + msg = ( + f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' + ) try: logger.info(msg) process_action(client, action_def, dry_run=ctx.params['dry_run']) - # pylint: disable=broad-except except Exception as err: exception_handler(action_def, err) logger.info('Action ID: %s, "%s" completed.', idx, action_def.action) logger.info('All actions completed.') -# pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals, line-too-long -@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html')) + +@click.command( + context_settings=context_settings(), + epilog=footer(__version__, tail='command-line.html'), +) @options_from_dict(OPTION_DEFAULTS) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(__version__, '-v', '--version', prog_name="curator") @click.pass_context def cli( - ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, - opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, - ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - loglevel, logfile, logformat, blacklist, dry_run, action_file + ctx, + config, + hosts, + cloud_id, + api_token, + id, + api_key, + username, + password, + bearer_auth, + opaque_id, + request_timeout, + http_compress, + verify_certs, + ca_certs, + client_cert, + client_key, + ssl_assert_hostname, + ssl_assert_fingerprint, + ssl_version, + master_only, + skip_version_test, + loglevel, + logfile, + logformat, + blacklist, + dry_run, + action_file, ): """ Curator for Elasticsearch indices The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. - + Command-line settings will always override YAML configuration settings. - Some less-frequently used client configuration options are now hidden. To see the full list, + Some less-frequently used client configuration options are now hidden. To see the + full list, + run: curator_cli -h diff --git a/curator/cli_singletons/delete.py b/curator/cli_singletons/delete.py index 6c788a9b..02deeac4 100644 --- a/curator/cli_singletons/delete.py +++ b/curator/cli_singletons/delete.py @@ -1,44 +1,57 @@ """Delete Index and Delete Snapshot Singletons""" + import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -#### Indices #### +# pylint: disable=R0913 + + +# Indices @click.command() -@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern') +@click.option( + '--search_pattern', + type=str, + default='_all', + help='Elasticsearch Index Search Pattern', +) @click.option( '--ignore_empty_list', is_flag=True, - help='Do not raise exception if there are no actionable indices' + help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, - show_default=True + show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', - required=True + required=True, ) @click.pass_context -def delete_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list): +def delete_indices( + ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list +): """ Delete Indices """ - # ctx.info_name is the name of the function or name specified in @click.command decorator + # ctx.info_name is the name of the function or name specified in @click.command + # decorator action = CLIAction( 'delete_indices', ctx.obj['configdict'], - {'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices}, + {'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices}, filter_list, - ignore_empty_list + ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) -#### Snapshots #### + +# Snapshots @click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') @@ -46,25 +59,30 @@ def delete_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, fi @click.option( '--ignore_empty_list', is_flag=True, - help='Do not raise exception if there are no actionable snapshots' + help='Do not raise exception if there are no actionable snapshots', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, - show_default=True + show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', - required=True + required=True, ) @click.pass_context def delete_snapshots( - ctx, repository, retry_count, retry_interval, - ignore_empty_list, allow_ilm_indices, filter_list - ): + ctx, + repository, + retry_count, + retry_interval, + ignore_empty_list, + allow_ilm_indices, + filter_list, +): """ Delete Snapshots """ @@ -73,13 +91,14 @@ def delete_snapshots( 'retry_interval': retry_interval, 'allow_ilm_indices': allow_ilm_indices, } - # ctx.info_name is the name of the function or name specified in @click.command decorator + # ctx.info_name is the name of the function or name specified in @click.command + # decorator action = CLIAction( 'delete_snapshots', ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, - repository=repository + repository=repository, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/defaults/filter_elements.py b/curator/defaults/filter_elements.py index 79992625..55f4c298 100644 --- a/curator/defaults/filter_elements.py +++ b/curator/defaults/filter_elements.py @@ -2,222 +2,265 @@ All member functions return a :class:`voluptuous.schema_builder.Schema` object """ -from six import string_types + from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required from curator.defaults import settings # pylint: disable=unused-argument, line-too-long + def aliases(**kwargs): """ - :returns: ``{Required('aliases'): Any(list, *string_types)}`` + :returns: {Required('aliases'): Any(list, str)} """ - return {Required('aliases'): Any(list, *string_types)} + return {Required('aliases'): Any(list, str)} + def allocation_type(**kwargs): """ - :returns: ``{Optional('allocation_type', default='require'): All(Any(*string_types), Any('require', 'include', 'exclude'))}`` + :returns: {Optional('allocation_type', default='require'): + All(Any(str), Any('require', 'include', 'exclude'))} """ return { Optional('allocation_type', default='require'): All( - Any(*string_types), Any('require', 'include', 'exclude')) + Any(str), Any('require', 'include', 'exclude') + ) } + def count(**kwargs): """ This setting is only used with the count filtertype and is required - :returns: ``{Required('count'): All(Coerce(int), Range(min=1))}`` + :returns: {Required('count'): All(Coerce(int), Range(min=1))} """ return {Required('count'): All(Coerce(int), Range(min=1))} + def date_from(**kwargs): """ This setting is only used with the period filtertype. - :returns: ``{Optional('date_from'): Any(*string_types)}`` + :returns: {Optional('date_from'): Any(str)} """ - return {Optional('date_from'): Any(*string_types)} + return {Optional('date_from'): Any(str)} + def date_from_format(**kwargs): """ This setting is only used with the period filtertype. - :returns: ``{Optional('date_from_format'): Any(*string_types)}`` + :returns: {Optional('date_from_format'): Any(str)} """ - return {Optional('date_from_format'): Any(*string_types)} + return {Optional('date_from_format'): Any(str)} + def date_to(**kwargs): """ This setting is only used with the period filtertype. - :returns: ``{Optional('date_to'): Any(*string_types)}`` + :returns: {Optional('date_to'): Any(str)} """ - return {Optional('date_to'): Any(*string_types)} + return {Optional('date_to'): Any(str)} + def date_to_format(**kwargs): """ This setting is only used with the period filtertype. - :returns: ``{Optional('date_to_format'): Any(*string_types)}`` + :returns: {Optional('date_to_format'): Any(str)} """ - return {Optional('date_to_format'): Any(*string_types)} + return {Optional('date_to_format'): Any(str)} + def direction(**kwargs): """ This setting is only used with the ``age`` filtertype. - :returns: ``{Required('direction'): Any('older', 'younger')}`` + :returns: {Required('direction'): Any('older', 'younger')} """ return {Required('direction'): Any('older', 'younger')} + def disk_space(**kwargs): """ This setting is only used with the ``space`` filtertype and is required - :returns: ``{Required('disk_space'): Any(Coerce(float))}`` + :returns: {Required('disk_space'): Any(Coerce(float))} """ return {Required('disk_space'): Any(Coerce(float))} + def epoch(**kwargs): """ This setting is only used with the ``age`` filtertype. - :returns: ``{Optional('epoch', default=None): Any(Coerce(int), None)}`` + :returns: {Optional('epoch', default=None): Any(Coerce(int), None)} """ return {Optional('epoch', default=None): Any(Coerce(int), None)} + def exclude(**kwargs): """ This setting is available in all filter types. The default ``val`` is ``True`` if ``exclude`` in ``kwargs``, otherwise ``False`` - :returns: ``{Optional('exclude', default=val): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: {Optional('exclude', default=val): + Any(bool, All(Any(str), Boolean()))} """ val = bool('exclude' in kwargs and kwargs['exclude']) # pylint: disable=no-value-for-parameter - return {Optional('exclude', default=val): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('exclude', default=val): Any(bool, All(Any(str), Boolean()))} + def field(**kwargs): """ This setting is only used with the ``age`` filtertype. - :returns: ``{Required('field'): Any(*string_types)}`` if ``kwargs['required']`` is ``True`` otherwise ``{Optional('field'): Any(*string_types)}`` + :returns: {Required('field'): Any(str)} if ``kwargs['required']`` is ``True`` + otherwise {Optional('field'): Any(str)} """ if 'required' in kwargs and kwargs['required']: - return {Required('field'): Any(*string_types)} - return {Optional('field'): Any(*string_types)} + return {Required('field'): Any(str)} + return {Optional('field'): Any(str)} + def intersect(**kwargs): """ - This setting is only used with the period filtertype when using field_stats, i.e. indices only. + This setting is only used with the period filtertype when using field_stats, i.e. + indices only. - :returns: ``{Optional('intersect', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: {Optional('intersect', default=False): + Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter - return {Optional('intersect', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('intersect', default=False): Any(bool, All(Any(str), Boolean()))} + def key(**kwargs): """ This setting is only used with the allocated filtertype. - :returns: ``{Required('key'): Any(*string_types)}`` + :returns: {Required('key'): Any(str)} """ - return {Required('key'): Any(*string_types)} + return {Required('key'): Any(str)} + def kind(**kwargs): """ This setting is only used with the pattern filtertype and is required - :returns: ``{Required('kind'): Any('prefix', 'suffix', 'timestring', 'regex')}`` + :returns: {Required('kind'): Any('prefix', 'suffix', 'timestring', 'regex')} """ return {Required('kind'): Any('prefix', 'suffix', 'timestring', 'regex')} + def max_num_segments(**kwargs): """ - :returns: ``{Required('max_num_segments'): All(Coerce(int), Range(min=1))}`` + :returns: {Required('max_num_segments'): All(Coerce(int), Range(min=1))} """ return {Required('max_num_segments'): All(Coerce(int), Range(min=1))} + def number_of_shards(**kwargs): """ - :returns: ``{Required('number_of_shards'): All(Coerce(int), Range(min=1))}`` + :returns: {Required('number_of_shards'): All(Coerce(int), Range(min=1))} """ return {Required('number_of_shards'): All(Coerce(int), Range(min=1))} + def pattern(**kwargs): """ - :returns: ``{Optional('pattern'): Any(*string_types)}`` + :returns: {Optional('pattern'): Any(str)} """ - return {Optional('pattern'): Any(*string_types)} + return {Optional('pattern'): Any(str)} + def period_type(**kwargs): """ This setting is only used with the period filtertype. - :returns: ``{Optional('period_type', default='relative'): Any('relative', 'absolute')}`` + :returns: {Optional('period_type', default='relative'): + Any('relative', 'absolute')} """ return {Optional('period_type', default='relative'): Any('relative', 'absolute')} + def range_from(**kwargs): """ - :returns: ``{Optional('range_from'): Coerce(int)}`` + :returns: {Optional('range_from'): Coerce(int)} """ return {Optional('range_from'): Coerce(int)} + def range_to(**kwargs): """ - :returns: ``{Optional('range_to'): Coerce(int)}`` + :returns: {Optional('range_to'): Coerce(int)} """ return {Optional('range_to'): Coerce(int)} + def reverse(**kwargs): """ Only used with ``space`` filtertype. Should be ignored if ```use_age``` is True - :returns: ``{Optional('reverse', default=True): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: {Optional('reverse', default=True): + Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter - return {Optional('reverse', default=True): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('reverse', default=True): Any(bool, All(Any(str), Boolean()))} + def shard_filter_behavior(**kwargs): """ This setting is only used with the shards filtertype and defaults to 'greater_than'. - :returns: ``{Optional('shard_filter_behavior', default='greater_than'): Any('greater_than', 'less_than', 'greater_than_or_equal', 'less_than_or_equal', 'equal')}`` + :returns: {Optional('shard_filter_behavior', default='greater_than'): + Any('greater_than', 'less_than', 'greater_than_or_equal', + 'less_than_or_equal', 'equal')} """ return { - Optional('shard_filter_behavior', default='greater_than'): - Any('greater_than', 'less_than', 'greater_than_or_equal', 'less_than_or_equal', 'equal') + Optional('shard_filter_behavior', default='greater_than'): Any( + 'greater_than', + 'less_than', + 'greater_than_or_equal', + 'less_than_or_equal', + 'equal', + ) } + def size_threshold(**kwargs): """ This setting is only used with the size filtertype and is required - :returns: ``{Required('size_threshold'): Any(Coerce(float))}`` + :returns: {Required('size_threshold'): Any(Coerce(float))} """ return {Required('size_threshold'): Any(Coerce(float))} + def size_behavior(**kwargs): """ This setting is only used with the size filtertype and defaults to 'primary'. - :returns: ``{Optional('size_behavior', default='primary'): Any('primary', 'total')}`` + :returns: {Optional('size_behavior', default='primary'): + Any('primary', 'total')} """ return {Optional('size_behavior', default='primary'): Any('primary', 'total')} + def source(**kwargs): """ - This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype when ``use_age`` is - set to True. + This setting is only used with the ``age`` filtertype, or with the ``space`` + filtertype when ``use_age`` is set to True. :ivar valuelist: If ``kwargs['action']`` is in :py:func:`curator.defaults.settings.snapshot_actions`, then it is - ``Any('name', 'creation_date')``, otherwise ``Any('name', 'creation_date', 'field_stats')`` - :returns: ``{Required('source'): valuelist}`` if ``kwargs['required']``, else - ``{Optional('source'): valuelist}`` + ``Any('name', 'creation_date')``, otherwise + ``Any('name', 'creation_date', 'field_stats')`` + :returns: {Required('source'): valuelist} if ``kwargs['required']``, else + {Optional('source'): valuelist} """ if 'action' in kwargs and kwargs['action'] in settings.snapshot_actions(): valuelist = Any('name', 'creation_date') @@ -226,97 +269,126 @@ def source(**kwargs): return {Required('source'): valuelist} return {Optional('source'): valuelist} + def state(**kwargs): """ This setting is only used with the state filtertype. - :returns: ``{Optional('state', default='SUCCESS'): Any('SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS')}`` + :returns: {Optional('state', default='SUCCESS'): + Any('SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS')} """ - return {Optional('state', default='SUCCESS'): Any('SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS')} + return { + Optional('state', default='SUCCESS'): Any( + 'SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS' + ) + } + def stats_result(**kwargs): """ This setting is only used with the ``age`` filtertype. - :returns: ``{Optional('stats_result', default='min_value'): Any('min_value', 'max_value')}`` + :returns: {Optional('stats_result', default='min_value'): + Any('min_value', 'max_value')} """ - return {Optional('stats_result', default='min_value'): Any('min_value', 'max_value')} + return { + Optional('stats_result', default='min_value'): Any('min_value', 'max_value') + } + def timestring(**kwargs): """ - This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if - ``use_age`` is set to ``True``. + This setting is only used with the ``age`` filtertype, or with the ``space`` + filtertype if ``use_age`` is set to ``True``. - :returns: ``{Required('timestring'): Any(*string_types)}`` if ``kwargs['required']`` else - ``{Optional('timestring', default=None): Any(None, *string_types)}`` + :returns: {Required('timestring'): Any(str)} if ``kwargs['required']`` else + {Optional('timestring', default=None): Any(None, str)} """ if 'required' in kwargs and kwargs['required']: - return {Required('timestring'): Any(*string_types)} - return {Optional('timestring', default=None): Any(None, *string_types)} + return {Required('timestring'): Any(str)} + return {Optional('timestring', default=None): Any(None, str)} + def threshold_behavior(**kwargs): """ - This setting is only used with the space and size filtertype and defaults to 'greater_than'. + This setting is only used with the space and size filtertype and defaults to + 'greater_than'. - :returns: ``{Optional('threshold_behavior', default='greater_than'): Any('greater_than', 'less_than')}`` + :returns: {Optional('threshold_behavior', default='greater_than'): + Any('greater_than', 'less_than')} """ - return {Optional('threshold_behavior', default='greater_than'): Any('greater_than', 'less_than')} + return { + Optional('threshold_behavior', default='greater_than'): Any( + 'greater_than', 'less_than' + ) + } + def unit(**kwargs): """ - This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if - ``use_age`` is set to ``True``. + This setting is only used with the ``age`` filtertype, or with the ``space`` + filtertype if ``use_age`` is set to ``True``. - :returns: ``{Required('unit'): Any('seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years')}`` + :returns: {Required('unit'): Any('seconds', 'minutes', 'hours', 'days', 'weeks', + 'months', 'years')} """ - return {Required('unit'): Any('seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years')} + return { + Required('unit'): Any( + 'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years' + ) + } + def unit_count(**kwargs): """ - This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if - ``use_age`` is set to ``True``. + This setting is only used with the ``age`` filtertype, or with the ``space`` + filtertype if ``use_age`` is set to ``True``. - :returns: ``{Required('unit_count'): Coerce(int)}`` + :returns: {Required('unit_count'): Coerce(int)} """ return {Required('unit_count'): Coerce(int)} + def unit_count_pattern(**kwargs): """ - This setting is used with the ``age`` filtertype to define whether the ``unit_count`` value is - taken from the configuration or read from the index name via a regular expression + This setting is used with the ``age`` filtertype to define whether the + ``unit_count`` value is taken from the configuration or read from the index + name via a regular expression - :returns: ``{Optional('unit_count_pattern'): Any(*string_types)}`` + :returns: {Optional('unit_count_pattern'): Any(str)} """ - return {Optional('unit_count_pattern'): Any(*string_types)} + return {Optional('unit_count_pattern'): Any(str)} + def use_age(**kwargs): """ Use of this setting requires the additional setting, ``source``. - :returns: ``{Optional('use_age', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: {Optional('use_age', default=False): + Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter - return {Optional('use_age', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('use_age', default=False): Any(bool, All(Any(str), Boolean()))} + def value(**kwargs): """ - This setting is only used with the ``pattern`` filtertype and is a required setting. There is a - separate value option associated with the - ``Allocation`` action, and the - ``allocated`` filtertype. + This setting is only used with the ``pattern`` filtertype and is a required + setting. There is a separate value option associated with the ``Allocation`` + action, and the ``allocated`` filtertype. - :returns: ``{Required('value'): Any(*string_types)}`` + :returns: {Required('value'): Any(str)} """ - return {Required('value'): Any(*string_types)} + return {Required('value'): Any(str)} + def week_starts_on(**kwargs): """ - :returns: ``{Optional('week_starts_on', default='sunday'): Any('Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None)}`` + :returns: {Optional('week_starts_on', default='sunday'): + Any('Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None)} """ return { Optional('week_starts_on', default='sunday'): Any( 'Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None ) } - - diff --git a/curator/defaults/option_defaults.py b/curator/defaults/option_defaults.py index ef4c3018..8bca8c98 100644 --- a/curator/defaults/option_defaults.py +++ b/curator/defaults/option_defaults.py @@ -1,160 +1,257 @@ """Action Option Schema definitions""" -from six import string_types + from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required -# pylint: disable=line-too-long,missing-docstring # pylint: disable=E1120 def allocation_type(): """ - :returns: ``{Optional('allocation_type', default='require'): All(Any(*string_types), Any('require', 'include', 'exclude'))}`` + :returns: + {Optional('allocation_type', default='require'): + All(Any(str), Any('require', 'include', 'exclude'))} """ - return {Optional('allocation_type', default='require'): All(Any(*string_types), Any('require', 'include', 'exclude'))} + return { + Optional('allocation_type', default='require'): All( + Any(str), Any('require', 'include', 'exclude') + ) + } + def allow_ilm_indices(): """ - :returns: ``{Optional('allow_ilm_indices', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('allow_ilm_indices', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('allow_ilm_indices', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('allow_ilm_indices', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + def conditions(): """ - :returns: ``{Optional('conditions'): {Optional('max_age'): Any(*string_types), Optional('max_docs'): Coerce(int), Optional('max_size'): Any(*string_types)}}`` + :returns: + {Optional('conditions'): {Optional('max_age'): + Any(str), Optional('max_docs'): + Coerce(int), Optional('max_size'): Any(str)}} """ - return {Optional('conditions'): {Optional('max_age'): Any(*string_types), Optional('max_docs'): Coerce(int), Optional('max_size'): Any(*string_types)}} + return { + Optional('conditions'): { + Optional('max_age'): Any(str), + Optional('max_docs'): Coerce(int), + Optional('max_size'): Any(str), + } + } + def continue_if_exception(): """ - :returns: ``{Optional('continue_if_exception', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('continue_if_exception', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('continue_if_exception', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('continue_if_exception', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + def count(): """ - :returns: ``{Required('count'): All(Coerce(int), Range(min=0, max=10))}`` + :returns: {Required('count'): All(Coerce(int), Range(min=0, max=10))} """ return {Required('count'): All(Coerce(int), Range(min=0, max=10))} + def delay(): """ - :returns: ``{Optional('delay', default=0): All(Coerce(float), Range(min=0.0, max=3600.0))}`` + :returns: + {Optional('delay', default=0): + All(Coerce(float), Range(min=0.0, max=3600.0))} """ - return {Optional('delay', default=0): All(Coerce(float), Range(min=0.0, max=3600.0))} + return { + Optional('delay', default=0): All(Coerce(float), Range(min=0.0, max=3600.0)) + } + def c2f_index_settings(): """ Only for the :py:class:`~.curator.actions.Cold2Frozen` action - :returns: ``{Optional('index_settings'): Any(None, dict)}`` + :returns: {Optional('index_settings'): Any(None, dict)} """ - return {Optional('index_settings', default=None): Any(None, dict)} + return {Optional('index_settings', default=None): Any(None, dict)} + def c2f_ignore_index_settings(): """ Only for the :py:class:`~.curator.actions.Cold2Frozen` action - :returns: ``{Optional('ignore_index_settings'): Any(None, list)}`` + :returns: {Optional('ignore_index_settings'): Any(None, list)} """ - return {Optional('ignore_index_settings', default=None): Any(None, list)} + return {Optional('ignore_index_settings', default=None): Any(None, list)} + def copy_aliases(): """ - :returns: ``{Optional('copy_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('copy_aliases', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('copy_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('copy_aliases', default=False): Any(bool, All(Any(str), Boolean())) + } + def delete_after(): """ - :returns: ``{Optional('delete_after', default=True): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('delete_after', default=True): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('delete_after', default=True): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('delete_after', default=True): Any(bool, All(Any(str), Boolean()))} + def delete_aliases(): """ - :returns: ``{Optional('delete_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('delete_aliases', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('delete_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('delete_aliases', default=False): Any(bool, All(Any(str), Boolean())) + } + def skip_flush(): """ - :returns: ``{Optional('skip_flush', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('skip_flush', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('skip_flush', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('skip_flush', default=False): Any(bool, All(Any(str), Boolean()))} + def disable_action(): """ - :returns: ``{Optional('disable_action', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('disable_action', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('disable_action', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('disable_action', default=False): Any(bool, All(Any(str), Boolean())) + } + def extra_settings(): """ - :returns: ``{Optional('extra_settings', default={}): dict}`` + :returns: {Optional('extra_settings', default={}): dict} """ return {Optional('extra_settings', default={}): dict} + def ignore_empty_list(): """ - :returns: ``{Optional('ignore_empty_list', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('ignore_empty_list', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('ignore_empty_list', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('ignore_empty_list', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + def ignore_existing(): """ - :returns: ``{Optional('ignore_existing', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('ignore_existing', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('ignore_existing', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('ignore_existing', default=False): Any(bool, All(Any(str), Boolean())) + } + def ignore_unavailable(): """ - :returns: ``{Optional('ignore_unavailable', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('ignore_unavailable', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('ignore_unavailable', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('ignore_unavailable', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + def include_aliases(): """ - :returns: ``{Optional('include_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('include_aliases', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('include_aliases', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('include_aliases', default=False): Any(bool, All(Any(str), Boolean())) + } + def include_global_state(action): """ - :returns: ``{Optional('include_global_state', default=default): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('include_global_state', default=default): + Any(bool, All(Any(str), Boolean()))} """ default = False if action == 'snapshot': default = True - return {Optional('include_global_state', default=default): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('include_global_state', default=default): Any( + bool, All(Any(str), Boolean()) + ) + } + def index_settings(): """ - :returns: ``{Required('index_settings'): {'index': dict}}`` + :returns: {Required('index_settings'): {'index': dict}} """ return {Required('index_settings'): {'index': dict}} + def indices(): """ - :returns: ``{Optional('indices', default=None): Any(None, list)}`` + :returns: {Optional('indices', default=None): Any(None, list)} """ return {Optional('indices', default=None): Any(None, list)} + def key(): """ - :returns: ``{Required('key'): Any(*string_types)}`` + :returns: {Required('key'): Any(str)} """ - return {Required('key'): Any(*string_types)} + return {Required('key'): Any(str)} + def max_num_segments(): """ - :returns: ``{Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))}`` + :returns: + {Required('max_num_segments'): + All(Coerce(int), Range(min=1, max=32768))} """ return {Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))} + # pylint: disable=unused-argument def max_wait(action): """ - :returns: ``{Optional('max_wait', default=defval): Any(-1, Coerce(int), None)}`` + :returns: {Optional('max_wait', default=defval): Any(-1, Coerce(int), None)} """ # The separation is here in case I want to change defaults later... defval = -1 @@ -164,116 +261,157 @@ def max_wait(action): # defval = -1 return {Optional('max_wait', default=defval): Any(-1, Coerce(int), None)} + def migration_prefix(): """ - :returns: ``{Optional('migration_prefix', default=''): Any(None, *string_types)}`` + :returns: {Optional('migration_prefix', default=''): Any(None, str)} """ - return {Optional('migration_prefix', default=''): Any(None, *string_types)} + return {Optional('migration_prefix', default=''): Any(None, str)} + def migration_suffix(): """ - :returns: ``{Optional('migration_suffix', default=''): Any(None, *string_types)}`` + :returns: {Optional('migration_suffix', default=''): Any(None, str)} """ - return {Optional('migration_suffix', default=''): Any(None, *string_types)} + return {Optional('migration_suffix', default=''): Any(None, str)} + def name(action): """ :returns: The proper name based on what action it is: - ``alias``, ``create_index``, ``rollover``: ``{Required('name'): Any(*string_types)}`` - ``snapshot``: ``{Optional('name', default='curator-%Y%m%d%H%M%S'): Any(*string_types)}`` - ``restore``: ``{Optional('name'): Any(*string_types)}`` + ``alias``, ``create_index``, ``rollover``: {Required('name'): Any(str)} + ``snapshot``: {Optional('name', default='curator-%Y%m%d%H%M%S'): Any(str)} + ``restore``: {Optional('name'): Any(str)} """ if action in ['alias', 'create_index', 'rollover']: - return {Required('name'): Any(*string_types)} + return {Required('name'): Any(str)} if action == 'snapshot': - return {Optional('name', default='curator-%Y%m%d%H%M%S'): Any(*string_types)} + return {Optional('name', default='curator-%Y%m%d%H%M%S'): Any(str)} if action == 'restore': - return {Optional('name'): Any(*string_types)} + return {Optional('name'): Any(str)} def new_index(): - return {Optional('new_index', default=None): Any(None, *string_types)} + """ + :returns: {Optional('new_index', default=None): Any(None, str)} + """ + return {Optional('new_index', default=None): Any(None, str)} + def node_filters(): """ - :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. + :returns: A :py:class:`voluptuous.schema_builder.Schema` object. + See code for more details. """ return { Optional('node_filters', default={}): { - Optional('permit_masters', default=False): Any(bool, All(Any(*string_types), Boolean())), - Optional('exclude_nodes', default=[]): Any(list, None) + Optional('permit_masters', default=False): Any( + bool, All(Any(str), Boolean()) + ), + Optional('exclude_nodes', default=[]): Any(list, None), } } + def number_of_replicas(): """ - :returns: ``{Optional('number_of_replicas', default=1): All(Coerce(int), Range(min=0, max=10))}`` + :returns: + {Optional('number_of_replicas', default=1): + All(Coerce(int), Range(min=0, max=10))} """ - return {Optional('number_of_replicas', default=1): All(Coerce(int), Range(min=0, max=10))} + return { + Optional('number_of_replicas', default=1): All( + Coerce(int), Range(min=0, max=10) + ) + } + def number_of_shards(): """ - :returns: ``{Optional('number_of_shards', default=1): All(Coerce(int), Range(min=1, max=99))}`` + :returns: + {Optional('number_of_shards', default=1): + All(Coerce(int), Range(min=1, max=99))} """ - return {Optional('number_of_shards', default=1): All(Coerce(int), Range(min=1, max=99))} + return { + Optional('number_of_shards', default=1): All(Coerce(int), Range(min=1, max=99)) + } + def partial(): """ - :returns: ``{Optional('partial', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('partial', default=False): Any(bool, All(Any(str), Boolean()))} """ - return {Optional('partial', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('partial', default=False): Any(bool, All(Any(str), Boolean()))} + def post_allocation(): """ - :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. + :returns: A :py:class:`voluptuous.schema_builder.Schema` object. + See code for more details. """ return { - Optional('post_allocation', default={}): - Any( - {}, - All( - { - Required('allocation_type', default='require'): All(Any(*string_types), Any('require', 'include', 'exclude')), - Required('key'): Any(*string_types), - Required('value', default=None): Any(None, *string_types) - } - ) - ) + Optional('post_allocation', default={}): Any( + {}, + All( + { + Required('allocation_type', default='require'): All( + Any(str), Any('require', 'include', 'exclude') + ), + Required('key'): Any(str), + Required('value', default=None): Any(None, str), + } + ), + ) } + def preserve_existing(): """ - :returns: ``{Optional('preserve_existing', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('preserve_existing', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('preserve_existing', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('preserve_existing', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + def refresh(): """ - :returns: ``{Optional('refresh', default=True): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('refresh', default=True): Any(bool, All(Any(str), Boolean()))} """ - return {Optional('refresh', default=True): Any(bool, All(Any(*string_types), Boolean()))} + return {Optional('refresh', default=True): Any(bool, All(Any(str), Boolean()))} + def remote_certificate(): """ - :returns: ``{Optional('remote_certificate', default=None): Any(None, *string_types)}`` + :returns: {Optional('remote_certificate', default=None): Any(None, str)} """ - return {Optional('remote_certificate', default=None): Any(None, *string_types)} + return {Optional('remote_certificate', default=None): Any(None, str)} + def remote_client_cert(): """ - :returns: ``{Optional('remote_client_cert', default=None): Any(None, *string_types)}`` + :returns: {Optional('remote_client_cert', default=None): Any(None, str)} """ - return {Optional('remote_client_cert', default=None): Any(None, *string_types)} + return {Optional('remote_client_cert', default=None): Any(None, str)} + def remote_client_key(): """ - :returns: ``{Optional('remote_client_key', default=None): Any(None, *string_types)}`` + :returns: {Optional('remote_client_key', default=None): Any(None, str)} """ - return {Optional('remote_client_key', default=None): Any(None, *string_types)} + return {Optional('remote_client_key', default=None): Any(None, str)} + def remote_filters(): """ - :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. + :returns: A :py:class:`voluptuous.schema_builder.Schema` object. + See code for more details. """ # This is really just a basic check here. The real check is in the # validate_actions() method in utils.py @@ -287,147 +425,192 @@ def remote_filters(): 'value': '.*', 'exclude': True, } - ] + ], ): Any(list, None) } + def rename_pattern(): """ - :returns: ``{Optional('rename_pattern'): Any(*string_types)}`` + :returns: {Optional('rename_pattern'): Any(str)} """ - return {Optional('rename_pattern'): Any(*string_types)} + return {Optional('rename_pattern'): Any(str)} + def rename_replacement(): """ - :returns: ``{Optional('rename_replacement'): Any(*string_types)}`` + :returns: {Optional('rename_replacement'): Any(str)} """ - return {Optional('rename_replacement'): Any(*string_types)} + return {Optional('rename_replacement'): Any(str)} + def repository(): """ - :returns: ``{Required('repository'): Any(*string_types)}`` + :returns: {Required('repository'): Any(str)} """ - return {Required('repository'): Any(*string_types)} + return {Required('repository'): Any(str)} + def request_body(): """ - :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. + :returns: A :py:class:`voluptuous.schema_builder.Schema` object. + See code for more details. """ return { Required('request_body'): { Optional('conflicts'): Any('proceed', 'abort'), Optional('max_docs'): Coerce(int), Required('source'): { - Required('index'): Any(Any(*string_types), list), + Required('index'): Any(Any(str), list), Optional('query'): dict, Optional('remote'): { - Optional('host'): Any(*string_types), - Optional('username'): Any(*string_types), - Optional('password'): Any(*string_types), - Optional('socket_timeout'): Any(*string_types), - Optional('connect_timeout'): Any(*string_types), - Optional('headers'): Any(*string_types), + Optional('host'): Any(str), + Optional('username'): Any(str), + Optional('password'): Any(str), + Optional('socket_timeout'): Any(str), + Optional('connect_timeout'): Any(str), + Optional('headers'): Any(str), }, Optional('size'): Coerce(int), Optional('_source'): Any(bool, Boolean()), }, Required('dest'): { - Required('index'): Any(*string_types), - Optional('version_type'): Any('internal', 'external', 'external_gt', 'external_gte'), - Optional('op_type'): Any(*string_types), - Optional('pipeline'): Any(*string_types), + Required('index'): Any(str), + Optional('version_type'): Any( + 'internal', 'external', 'external_gt', 'external_gte' + ), + Optional('op_type'): Any(str), + Optional('pipeline'): Any(str), }, Optional('script'): { - Optional('source'): Any(*string_types), - Optional('lang'): Any('painless', 'expression', 'mustache', 'java') + Optional('source'): Any(str), + Optional('lang'): Any('painless', 'expression', 'mustache', 'java'), }, } } + def requests_per_second(): """ - :returns: ``{Optional('requests_per_second', default=-1): Any(-1, Coerce(int), None)}`` + :returns: + {Optional('requests_per_second', default=-1): Any(-1, Coerce(int), None)} """ return {Optional('requests_per_second', default=-1): Any(-1, Coerce(int), None)} + def retry_count(): """ - :returns: ``{Optional('retry_count', default=3): All(Coerce(int), Range(min=0, max=100))}`` + :returns: + {Optional('retry_count', default=3): + All(Coerce(int), Range(min=0, max=100))} """ return {Optional('retry_count', default=3): All(Coerce(int), Range(min=0, max=100))} + def retry_interval(): """ - :returns: ``{Optional('retry_interval', default=120): All(Coerce(int), Range(min=1, max=600))}`` + :returns: + {Optional('retry_interval', default=120): + All(Coerce(int), Range(min=1, max=600))} """ - return {Optional('retry_interval', default=120): All(Coerce(int), Range(min=1, max=600))} + return { + Optional('retry_interval', default=120): All(Coerce(int), Range(min=1, max=600)) + } + def routing_type(): """ - :returns: ``{Required('routing_type'): Any('allocation', 'rebalance')}`` + :returns: {Required('routing_type'): Any('allocation', 'rebalance')} """ return {Required('routing_type'): Any('allocation', 'rebalance')} + def cluster_routing_setting(): """ - :returns: ``{Required('setting'): Any('enable')}`` + :returns: {Required('setting'): Any('enable')} """ return {Required('setting'): Any('enable')} + def cluster_routing_value(): """ - :returns: ``{Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')}`` + :returns: + {Required('value'): + Any('all', 'primaries', 'none', 'new_primaries', 'replicas')} """ - return {Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')} + return { + Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas') + } + def search_pattern(): """ - :returns: ``{Optional('search_pattern', default='_all'): Any(*string_types)}`` + :returns: {Optional('search_pattern', default='_all'): Any(str)} """ - return {Optional('search_pattern', default='_all'): Any(*string_types)} + return {Optional('search_pattern', default='_all'): Any(str)} + def shrink_node(): """ - :returns: ``{Required('shrink_node'): Any(*string_types)}`` + :returns: {Required('shrink_node'): Any(str)} """ - return {Required('shrink_node'): Any(*string_types)} + return {Required('shrink_node'): Any(str)} + def shrink_prefix(): """ - :returns: ``{Optional('shrink_prefix', default=''): Any(None, *string_types)}`` + :returns: {Optional('shrink_prefix', default=''): Any(None, str)} """ - return {Optional('shrink_prefix', default=''): Any(None, *string_types)} + return {Optional('shrink_prefix', default=''): Any(None, str)} + def shrink_suffix(): """ - :returns: ``{Optional('shrink_suffix', default='-shrink'): Any(None, *string_types)}`` + :returns: {Optional('shrink_suffix', default='-shrink'): Any(None, str)} """ - return {Optional('shrink_suffix', default='-shrink'): Any(None, *string_types)} + return {Optional('shrink_suffix', default='-shrink'): Any(None, str)} + def skip_repo_fs_check(): """ - :returns: ``{Optional('skip_repo_fs_check', default=True): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('skip_repo_fs_check', default=True): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('skip_repo_fs_check', default=True): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('skip_repo_fs_check', default=True): Any( + bool, All(Any(str), Boolean()) + ) + } + def slices(): """ - :returns: ``{Optional('slices', default=1): Any(All(Coerce(int), Range(min=1, max=500)), None)}`` + :returns: + {Optional('slices', default=1): + Any(All(Coerce(int), Range(min=1, max=500)), None)} """ - return {Optional('slices', default=1): Any(All(Coerce(int), Range(min=1, max=500)), None)} + return { + Optional('slices', default=1): Any( + All(Coerce(int), Range(min=1, max=500)), None + ) + } + def timeout(action): """ - :returns: ``{Optional('timeout', default=defval): Any(Coerce(int), None)}`` + :returns: {Optional('timeout', default=defval): Any(Coerce(int), None)} """ # if action == 'reindex': defval = 60 return {Optional('timeout', default=defval): Any(Coerce(int), None)} + def timeout_override(action): """ - :returns: ``{Optional('timeout_override', default=defval): Any(Coerce(int), None)}`` where - ``defval`` is determined by the action. + :returns: + {Optional('timeout_override', default=defval): Any(Coerce(int), None)} + where ``defval`` is determined by the action. ``['forcemerge', 'restore', 'snapshot']`` = ``21600`` @@ -445,45 +628,73 @@ def timeout_override(action): defval = None return {Optional('timeout_override', default=defval): Any(Coerce(int), None)} + def value(): """ - :returns: ``{Required('value', default=None): Any(None, *string_types)}`` + :returns: {Required('value', default=None): Any(None, str)} """ - return {Required('value', default=None): Any(None, *string_types)} + return {Required('value', default=None): Any(None, str)} + def wait_for_active_shards(action): """ - :returns: ``{Optional('wait_for_active_shards', default=defval): Any(Coerce(int), 'all', None)}`` - where ``defval`` defaults to 0, but changes to 1 for the ``reindex`` and ``shrink`` actions. + :returns: + {Optional('wait_for_active_shards', default=defval): + Any(Coerce(int), 'all', None)} + where ``defval`` defaults to 0, but changes to 1 for the ``reindex`` and + ``shrink`` actions. """ defval = 0 if action in ['reindex', 'shrink']: defval = 1 - return {Optional('wait_for_active_shards', default=defval): Any(Coerce(int), 'all', None)} + return { + Optional('wait_for_active_shards', default=defval): Any( + Coerce(int), 'all', None + ) + } + def wait_for_completion(action): """ - :returns: ``{Optional('wait_for_completion', default=defval): Any(bool, All(Any(*string_types), Boolean()))}`` - where ``defval`` defaults to True, but changes to False if action is ``allocation``, - ``cluster_routing``, or ``replicas``. + :returns: + {Optional('wait_for_completion', default=defval): + Any(bool, All(Any(str), Boolean()))} + where ``defval`` defaults to True, but changes to False if action is + ``allocation``, ``cluster_routing``, or ``replicas``. """ # if action in ['cold2frozen', 'reindex', 'restore', 'snapshot']: defval = True if action in ['allocation', 'cluster_routing', 'replicas']: defval = False - return {Optional('wait_for_completion', default=defval): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('wait_for_completion', default=defval): Any( + bool, All(Any(str), Boolean()) + ) + } + def wait_for_rebalance(): """ - :returns: ``{Optional('wait_for_rebalance', default=True): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('wait_for_rebalance', default=True): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('wait_for_rebalance', default=True): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('wait_for_rebalance', default=True): Any( + bool, All(Any(str), Boolean()) + ) + } + def wait_interval(action): """ - :returns: ``{Optional('wait_interval', default=defval): Any(All(Coerce(int), Range(min=minval, max=maxval)), None)}`` - where ``minval`` = ``1``, ``maxval`` = ``30``, and ``defval`` is ``3``, unless the action - is one of ``['restore', 'snapshot', 'reindex', 'shrink']``, and then ``defval`` is ``9``. + :returns: + {Optional('wait_interval', default=defval): + Any(All(Coerce(int), Range(min=minval, max=maxval)), None)} + where ``minval`` = ``1``, ``maxval`` = ``30``, and ``defval`` is ``3``, + unless the action is one of + ``['restore', 'snapshot', 'reindex', 'shrink']``, and then ``defval`` + is ``9``. """ minval = 1 maxval = 30 @@ -491,10 +702,21 @@ def wait_interval(action): defval = 3 if action in ['restore', 'snapshot', 'reindex', 'shrink']: defval = 9 - return {Optional('wait_interval', default=defval): Any(All(Coerce(int), Range(min=minval, max=maxval)), None)} + return { + Optional('wait_interval', default=defval): Any( + All(Coerce(int), Range(min=minval, max=maxval)), None + ) + } + def warn_if_no_indices(): """ - :returns: ``{Optional('warn_if_no_indices', default=False): Any(bool, All(Any(*string_types), Boolean()))}`` + :returns: + {Optional('warn_if_no_indices', default=False): + Any(bool, All(Any(str), Boolean()))} """ - return {Optional('warn_if_no_indices', default=False): Any(bool, All(Any(*string_types), Boolean()))} + return { + Optional('warn_if_no_indices', default=False): Any( + bool, All(Any(str), Boolean()) + ) + } diff --git a/curator/defaults/settings.py b/curator/defaults/settings.py index 722428d6..d7becc25 100644 --- a/curator/defaults/settings.py +++ b/curator/defaults/settings.py @@ -1,9 +1,11 @@ """Utilities/Helpers for defaults and schemas""" + from os import path -from six import string_types from voluptuous import Any, Boolean, Coerce, Optional from curator.exceptions import CuratorException +# pylint: disable=E1120 + CURATOR_DOCS = 'https://www.elastic.co/guide/en/elasticsearch/client/curator' CLICK_DRYRUN = { 'dry-run': {'help': 'Do not perform any changes.', 'is_flag': True}, @@ -11,6 +13,7 @@ # Click specifics + def footer(version, tail='index.html'): """ Generate a footer linking to Curator docs based on Curator version @@ -18,7 +21,7 @@ def footer(version, tail='index.html'): :param version: The Curator version :type version: str - + :returns: An epilog/footer suitable for Click """ if not isinstance(version, str): @@ -32,21 +35,24 @@ def footer(version, tail='index.html'): raise CuratorException(msg) from exc return f'Learn more at {CURATOR_DOCS}/{majmin}/{tail}' + # Default Config file location def default_config_file(): """ :returns: The default configuration file location: - ``path.join(path.expanduser('~'), '.curator', 'curator.yml')`` + path.join(path.expanduser('~'), '.curator', 'curator.yml') """ default = path.join(path.expanduser('~'), '.curator', 'curator.yml') if path.isfile(default): return default + # Default filter patterns (regular expressions) def regex_map(): """ - :returns: A dictionary of pattern filter 'kind's with their associated regular expression: - ``{'timestring': r'^.*{0}.*$', 'regex': r'{0}', 'prefix': r'^{0}.*$', 'suffix': r'^.*{0}$'}`` + :returns: A dictionary of pattern filter 'kind's with their associated regular + expression: {'timestring': r'^.*{0}.*$', 'regex': r'{0}', + 'prefix': r'^{0}.*$', 'suffix': r'^.*{0}$'} """ return { 'timestring': r'^.*{0}.*$', @@ -55,10 +61,12 @@ def regex_map(): 'suffix': r'^.*{0}$', } + def date_regex(): """ - :returns: A dictionary/map of the strftime string characters and their string lengths: - ``{'Y':'4', 'G':'4', 'y':'2', 'm':'2', 'W':'2', 'V':'2', 'U':'2', 'd':'2', 'H':'2', 'M':'2', 'S':'2', 'j':'3'}`` + :returns: A dictionary/map of the strftime string characters and their string + lengths: {'Y':'4', 'G':'4', 'y':'2', 'm':'2', 'W':'2', 'V':'2', 'U':'2', + 'd':'2', 'H':'2', 'M':'2', 'S':'2', 'j':'3'} """ return { 'Y': '4', @@ -75,18 +83,24 @@ def date_regex(): 'j': '3', } + # Actions + def cluster_actions(): """ - :returns: A list of supported cluster actions (right now, that's only ``['cluster_routing']``) + :returns: A list of supported cluster actions (right now, that's only + ['cluster_routing']) """ return ['cluster_routing'] + def index_actions(): """ :returns: The list of supported index actions: - ``[ 'alias', 'allocation', 'close', 'create_index', 'delete_indices', 'forcemerge', 'index_settings', 'open', 'reindex', 'replicas', 'rollover', 'shrink', 'snapshot']`` + [ 'alias', 'allocation', 'close', 'create_index', 'delete_indices', + 'forcemerge', 'index_settings', 'open', 'reindex', 'replicas', + 'rollover', 'shrink', 'snapshot'] """ return [ 'alias', @@ -105,22 +119,27 @@ def index_actions(): 'snapshot', ] + def snapshot_actions(): """ - :returns: The list of supported snapshot actions: ``['delete_snapshots', 'restore']`` + :returns: The list of supported snapshot actions: ['delete_snapshots', 'restore'] """ return ['delete_snapshots', 'restore'] + def all_actions(): """ :returns: A sorted list of all supported actions: cluster, index, and snapshot """ return sorted(cluster_actions() + index_actions() + snapshot_actions()) + def index_filtertypes(): """ :returns: The list of supported index filter types: - ``['alias', 'allocated', 'age', 'closed', 'count', 'empty', 'forcemerged', 'ilm', 'kibana', 'none', 'opened', 'pattern', 'period', 'space', 'shards', 'size']`` + ['alias', 'allocated', 'age', 'closed', 'count', 'empty', 'forcemerged', + 'ilm', 'kibana', 'none', 'opened', 'pattern', 'period', 'space', + 'shards', 'size'] """ return [ @@ -142,22 +161,28 @@ def index_filtertypes(): 'size', ] + def snapshot_filtertypes(): """ - :returns: The list of supported snapshot filter types: ``['age', 'count', 'none', 'pattern', 'period', 'state']`` + :returns: The list of supported snapshot filter types: ['age', 'count', 'none', + 'pattern', 'period', 'state'] """ return ['age', 'count', 'none', 'pattern', 'period', 'state'] + def all_filtertypes(): """ :returns: A sorted list of all supported filter types (both snapshot and index) """ return sorted(list(set(index_filtertypes() + snapshot_filtertypes()))) + def default_options(): """ :returns: The default values for these options: - ``{'allow_ilm_indices': False, 'continue_if_exception': False, 'disable_action': False, 'ignore_empty_list': False, 'timeout_override': None}`` + {'allow_ilm_indices': False, 'continue_if_exception': False, + 'disable_action': False, 'ignore_empty_list': False, + 'timeout_override': None} """ return { 'allow_ilm_indices': False, @@ -167,6 +192,7 @@ def default_options(): 'timeout_override': None, } + def default_filters(): """ If no filters are set, add a 'none' filter @@ -175,46 +201,47 @@ def default_filters(): """ return {'filters': [{'filtertype': 'none'}]} + def structural_filter_elements(): """ :returns: Barebones schemas for initial validation of filters """ - # pylint: disable=E1120 + return { - Optional('aliases'): Any(list, *string_types), - Optional('allocation_type'): Any(*string_types), + Optional('aliases'): Any(list, str), + Optional('allocation_type'): Any(str), Optional('count'): Coerce(int), - Optional('date_from'): Any(None, *string_types), - Optional('date_from_format'): Any(None, *string_types), - Optional('date_to'): Any(None, *string_types), - Optional('date_to_format'): Any(None, *string_types), - Optional('direction'): Any(*string_types), + Optional('date_from'): Any(None, str), + Optional('date_from_format'): Any(None, str), + Optional('date_to'): Any(None, str), + Optional('date_to_format'): Any(None, str), + Optional('direction'): Any(str), Optional('disk_space'): float, Optional('epoch'): Any(Coerce(int), None), - Optional('exclude'): Any(None, bool, int, *string_types), - Optional('field'): Any(None, *string_types), - Optional('intersect'): Any(None, bool, int, *string_types), - Optional('key'): Any(*string_types), - Optional('kind'): Any(*string_types), + Optional('exclude'): Any(None, bool, int, str), + Optional('field'): Any(None, str), + Optional('intersect'): Any(None, bool, int, str), + Optional('key'): Any(str), + Optional('kind'): Any(str), Optional('max_num_segments'): Coerce(int), Optional('number_of_shards'): Coerce(int), - Optional('pattern'): Any(*string_types), - Optional('period_type'): Any(*string_types), - Optional('reverse'): Any(None, bool, int, *string_types), + Optional('pattern'): Any(str), + Optional('period_type'): Any(str), + Optional('reverse'): Any(None, bool, int, str), Optional('range_from'): Coerce(int), Optional('range_to'): Coerce(int), - Optional('shard_filter_behavior'): Any(*string_types), - Optional('size_behavior'): Any(*string_types), + Optional('shard_filter_behavior'): Any(str), + Optional('size_behavior'): Any(str), Optional('size_threshold'): Any(Coerce(float)), - Optional('source'): Any(*string_types), - Optional('state'): Any(*string_types), - Optional('stats_result'): Any(None, *string_types), - Optional('timestring'): Any(None, *string_types), - Optional('threshold_behavior'): Any(*string_types), - Optional('unit'): Any(*string_types), + Optional('source'): Any(str), + Optional('state'): Any(str), + Optional('stats_result'): Any(None, str), + Optional('timestring'): Any(None, str), + Optional('threshold_behavior'): Any(str), + Optional('unit'): Any(str), Optional('unit_count'): Coerce(int), - Optional('unit_count_pattern'): Any(*string_types), + Optional('unit_count_pattern'): Any(str), Optional('use_age'): Boolean(), - Optional('value'): Any(int, float, bool, *string_types), - Optional('week_starts_on'): Any(None, *string_types), + Optional('value'): Any(int, float, bool, str), + Optional('week_starts_on'): Any(None, str), } diff --git a/curator/helpers/date_ops.py b/curator/helpers/date_ops.py index 18b4a40d..de6970cf 100644 --- a/curator/helpers/date_ops.py +++ b/curator/helpers/date_ops.py @@ -1,4 +1,5 @@ """Curator date and time functions""" + import logging import random import re @@ -9,7 +10,8 @@ from curator.exceptions import ConfigurationError from curator.defaults.settings import date_regex -class TimestringSearch(object): + +class TimestringSearch: """ An object to allow repetitive search against a string, ``searchme``, without having to repeatedly recreate the regex. @@ -17,6 +19,7 @@ class TimestringSearch(object): :param timestring: An ``strftime`` pattern :type timestring: :py:func:`~.time.strftime` """ + def __init__(self, timestring): # pylint: disable=consider-using-f-string regex = r'(?P{0})'.format(get_date_regex(timestring)) @@ -33,8 +36,8 @@ def get_epoch(self, searchme): :param searchme: A string to be matched against :py:attr:`pattern` that matches :py:attr:`timestring` - :returns: The epoch timestamp extracted from ``searchme`` by regex matching against - :py:attr:`pattern` + :returns: The epoch timestamp extracted from ``searchme`` by regex matching + against :py:attr:`pattern` :rtype: int """ match = self.pattern.search(searchme) @@ -45,21 +48,23 @@ def get_epoch(self, searchme): def absolute_date_range( - unit, date_from, date_to, - date_from_format=None, date_to_format=None - ): + unit, date_from, date_to, date_from_format=None, date_to_format=None +): """ - This function calculates a date range with an absolute time stamp for both the start time and - the end time. These dates are converted to epoch time. The parameter ``unit`` is used when the - same simplified date is used for both ``date_from`` and ``date_to`` to calculate the duration. - For example, if ``unit`` is ``months``, and ``date_from`` and ``date_to`` are both ``2017.01``, - then the entire month of January 2017 will be the absolute date range. + This function calculates a date range with an absolute time stamp for both the + start time and the end time. These dates are converted to epoch time. The parameter + ``unit`` is used when the same simplified date is used for both ``date_from`` and + ``date_to`` to calculate the duration. For example, if ``unit`` is ``months``, and + ``date_from`` and ``date_to`` are both ``2017.01``, then the entire month of + January 2017 will be the absolute date range. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param date_from: The simplified date for the start of the range :param date_to: The simplified date for the end of the range. - :param date_from_format: The :py:func:`~.time.strftime` string used to parse ``date_from`` - :param date_to_format: The :py:func:`~.time.strftime` string used to parse ``date_to`` + :param date_from_format: The :py:func:`~.time.strftime` string used to parse + ``date_from`` + :param date_to_format: The :py:func:`~.time.strftime` string used to parse + ``date_to`` :type unit: str :type date_from: str @@ -71,7 +76,15 @@ def absolute_date_range( :rtype: tuple """ logger = logging.getLogger(__name__) - acceptable_units = ['seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years'] + acceptable_units = [ + 'seconds', + 'minutes', + 'hours', + 'days', + 'weeks', + 'months', + 'years', + ] if unit not in acceptable_units: raise ConfigurationError(f'"unit" must be one of: {acceptable_units}') if not date_from_format or not date_to_format: @@ -81,15 +94,15 @@ def absolute_date_range( logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch)) except Exception as err: raise ConfigurationError( - f'Unable to parse "date_from" {date_from} and "date_from_format" {date_from_format}. ' - f'Error: {err}' + f'Unable to parse "date_from" {date_from} and "date_from_format" ' + f'{date_from_format}. Error: {err}' ) from err try: end_date = get_datetime(date_to, date_to_format) except Exception as err: raise ConfigurationError( - f'Unable to parse "date_to" {date_to} and "date_to_format" {date_to_format}. ' - f'Error: {err}' + f'Unable to parse "date_to" {date_to} and "date_to_format" ' + f'{date_to_format}. Error: {err}' ) from err # We have to iterate to one more month, and then subtract a second to get # the last day of the correct month @@ -114,25 +127,28 @@ def absolute_date_range( # We use -1 as point of reference normally subtracts from the epoch # and we need to add to it, so we'll make it subtract a negative value. # Then, as before, subtract 1 to get the end of the period - end_epoch = get_point_of_reference( - unit, -1, epoch=datetime_to_epoch(end_date)) -1 + end_epoch = ( + get_point_of_reference(unit, -1, epoch=datetime_to_epoch(end_date)) - 1 + ) logger.debug('End ISO8601 = %s', epoch2iso(end_epoch)) return (start_epoch, end_epoch) + def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): """ - This function calculates a date range with a distinct epoch time stamp of both the start time - and the end time in counts of ``unit`` relative to the time at execution, if ``epoch`` is not - set. + This function calculates a date range with a distinct epoch time stamp of both the + start time and the end time in counts of ``unit`` relative to the time at + execution, if ``epoch`` is not set. - If ``unit`` is ``weeks``, you can also determine when a week begins using ``week_starts_on``, - which can be either ``sunday`` or ``monday``. + If ``unit`` is ``weeks``, you can also determine when a week begins using + ``week_starts_on``, which can be either ``sunday`` or ``monday``. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param range_from: Count of ``unit`` in the past/future is the origin? :param range_to: Count of ``unit`` in the past/future is the end point? - :param epoch: An epoch timestamp used to establish a point of reference for calculations. + :param epoch: An epoch timestamp used to establish a point of reference for + calculations. :param week_starts_on: Either ``sunday`` or ``monday``. Default is ``sunday`` :type unit: str @@ -149,7 +165,9 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): if unit not in acceptable_units: raise ConfigurationError(f'"unit" must be one of: {acceptable_units}') if not range_to >= range_from: - raise ConfigurationError('"range_to" must be greater than or equal to "range_from"') + raise ConfigurationError( + '"range_to" must be greater than or equal to "range_from"' + ) if not epoch: epoch = time.time() epoch = fix_epoch(epoch) @@ -161,19 +179,23 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): # These if statements help get the start date or start_delta if unit == 'hours': point_of_ref = datetime( - raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, - raw_point_of_ref.hour, 0, 0 + raw_point_of_ref.year, + raw_point_of_ref.month, + raw_point_of_ref.day, + raw_point_of_ref.hour, + 0, + 0, ) start_delta = timedelta(hours=origin) if unit == 'days': point_of_ref = datetime( - raw_point_of_ref.year, raw_point_of_ref.month, - raw_point_of_ref.day, 0, 0, 0 - ) + raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, 0, 0, 0 + ) start_delta = timedelta(days=origin) if unit == 'weeks': point_of_ref = datetime( - raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, 0, 0, 0) + raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, 0, 0, 0 + ) sunday = False if week_starts_on.lower() == 'sunday': sunday = True @@ -184,7 +206,9 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): logger.debug('Weekday = %s', weekday) start_delta = timedelta(days=weekday, weeks=origin) if unit == 'months': - point_of_ref = datetime(raw_point_of_ref.year, raw_point_of_ref.month, 1, 0, 0, 0) + point_of_ref = datetime( + raw_point_of_ref.year, raw_point_of_ref.month, 1, 0, 0, 0 + ) year = raw_point_of_ref.year month = raw_point_of_ref.month if origin > 0: @@ -233,11 +257,11 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): else: # This lets us use an existing method to simply add unit * count seconds # to get hours, days, or weeks, as they don't change - end_epoch = get_point_of_reference( - unit, count * -1, epoch=start_epoch) -1 + end_epoch = get_point_of_reference(unit, count * -1, epoch=start_epoch) - 1 logger.debug('End ISO8601 = %s', epoch2iso(end_epoch)) return (start_epoch, end_epoch) + def datetime_to_epoch(mydate): """ Converts datetime into epoch seconds @@ -251,6 +275,7 @@ def datetime_to_epoch(mydate): tdelta = mydate - datetime(1970, 1, 1) return tdelta.seconds + tdelta.days * 24 * 3600 + def epoch2iso(epoch: int) -> str: """ Return an ISO8601 value for epoch @@ -263,44 +288,49 @@ def epoch2iso(epoch: int) -> str: """ # Because Python 3.12 now requires non-naive timezone declarations, we must change. # - ### Example: - ### epoch == 1491256800 - ### - ### The old way: - ###datetime.utcfromtimestamp(epoch) - ### datetime.datetime(2017, 4, 3, 22, 0).isoformat() - ### Result: 2017-04-03T22:00:00 - ### - ### The new way: - ### datetime.fromtimestamp(epoch, timezone.utc) - ### datetime.datetime(2017, 4, 3, 22, 0, tzinfo=datetime.timezone.utc).isoformat() - ### Result: 2017-04-03T22:00:00+00:00 - ### - ### End Example + # ## Example: + # ## epoch == 1491256800 + # ## + # ## The old way: + # ##datetime.utcfromtimestamp(epoch) + # ## datetime.datetime(2017, 4, 3, 22, 0).isoformat() + # ## Result: 2017-04-03T22:00:00 + # ## + # ## The new way: + # ## datetime.fromtimestamp(epoch, timezone.utc) + # ## datetime.datetime( + # ## 2017, 4, 3, 22, 0, tzinfo=datetime.timezone.utc).isoformat() + # ## Result: 2017-04-03T22:00:00+00:00 + # ## + # ## End Example # - # Note that the +00:00 is appended now where we affirmatively declare the UTC timezone + # Note that the +00:00 is appended now where we affirmatively declare the UTC + # timezone # - # As a result, we will use this function to prune away the timezone if it is +00:00 and replace - # it with Z, which is shorter Zulu notation for UTC (which Elasticsearch uses) + # As a result, we will use this function to prune away the timezone if it is +00:00 + # and replace it with Z, which is shorter Zulu notation for UTC (which + # Elasticsearch uses) # - # We are MANUALLY, FORCEFULLY declaring timezone.utc, so it should ALWAYS be +00:00, but could - # in theory sometime show up as a Z, so we test for that. + # We are MANUALLY, FORCEFULLY declaring timezone.utc, so it should ALWAYS be + # +00:00, but could in theory sometime show up as a Z, so we test for that. parts = datetime.fromtimestamp(epoch, timezone.utc).isoformat().split('+') if len(parts) == 1: if parts[0][-1] == 'Z': - return parts[0] # Our ISO8601 already ends with a Z for Zulu/UTC time - return f'{parts[0]}Z' # It doesn't end with a Z so we put one there + return parts[0] # Our ISO8601 already ends with a Z for Zulu/UTC time + return f'{parts[0]}Z' # It doesn't end with a Z so we put one there if parts[1] == '00:00': - return f'{parts[0]}Z' # It doesn't end with a Z so we put one there - return f'{parts[0]}+{parts[1]}' # Fallback publishes the +TZ, whatever that was + return f'{parts[0]}Z' # It doesn't end with a Z so we put one there + return f'{parts[0]}+{parts[1]}' # Fallback publishes the +TZ, whatever that was + def fix_epoch(epoch): """ - Fix value of ``epoch`` to be the count since the epoch in seconds only, which should be 10 or - fewer digits long. + Fix value of ``epoch`` to be the count since the epoch in seconds only, which + should be 10 or fewer digits long. - :param epoch: An epoch timestamp, in epoch + milliseconds, or microsecond, or even nanoseconds. + :param epoch: An epoch timestamp, in epoch + milliseconds, or microsecond, or even + nanoseconds. :type epoch: int :returns: An epoch timestamp in seconds only, based on ``epoch`` @@ -310,7 +340,9 @@ def fix_epoch(epoch): # No decimals allowed epoch = int(epoch) except Exception as err: - raise ValueError(f'Bad epoch value. Unable to convert {epoch} to int. {err}') from err + raise ValueError( + f'Bad epoch value. Unable to convert {epoch} to int. {err}' + ) from err # If we're still using this script past January, 2038, we have bigger # problems than my hacky math here... @@ -318,19 +350,21 @@ def fix_epoch(epoch): # Epoch is fine, no changes pass elif len(str(epoch)) > 10 and len(str(epoch)) <= 13: - epoch = int(epoch/1000) + epoch = int(epoch / 1000) else: orders_of_magnitude = len(str(epoch)) - 10 powers_of_ten = 10**orders_of_magnitude - epoch = int(epoch/powers_of_ten) + epoch = int(epoch / powers_of_ten) return epoch + def get_date_regex(timestring): """ :param timestring: An ``strftime`` pattern :type timestring: :py:func:`~.time.strftime` - :returns: A regex string based on a provided :py:func:`~.time.strftime` ``timestring``. + :returns: A regex string based on a provided :py:func:`~.time.strftime` + ``timestring``. :rtype: str """ logger = logging.getLogger(__name__) @@ -350,12 +384,14 @@ def get_date_regex(timestring): logger.debug('regex = %s', regex) return regex + def get_datemath(client, datemath, random_element=None): """ :param client: A client connection object :param datemath: An elasticsearch datemath string - :param random_element: This allows external randomization of the name and is only useful for - tests so that you can guarantee the output because you provided the random portion. + :param random_element: This allows external randomization of the name and is only + useful for tests so that you can guarantee the output because you provided the + random portion. :type client: :py:class:`~.elasticsearch.Elasticsearch` :type datemath: :py:class:`~.datemath.datemath` @@ -366,9 +402,8 @@ def get_datemath(client, datemath, random_element=None): """ logger = logging.getLogger(__name__) if random_element is None: - random_prefix = ( - 'curator_get_datemath_function_' + - ''.join(random.choice(string.ascii_lowercase) for _ in range(32)) + random_prefix = 'curator_get_datemath_function_' + ''.join( + random.choice(string.ascii_lowercase) for _ in range(32) ) else: random_prefix = 'curator_get_datemath_function_' + random_element @@ -398,6 +433,7 @@ def get_datemath(client, datemath, random_element=None): f'or has invalid index name characters.' ) from exc + def get_datetime(index_timestamp, timestring): """ :param index_timestamp: The index timestamp @@ -406,7 +442,8 @@ def get_datetime(index_timestamp, timestring): :type index_timestamp: str :type timestring: :py:func:`~.time.strftime` - :returns: The datetime extracted from the index name, which is the index creation time. + :returns: The datetime extracted from the index name, which is the index creation + time. :rtype: :py:class:`~.datetime.datetime` """ # Compensate for week of year by appending '%w' to the timestring @@ -420,7 +457,7 @@ def get_datetime(index_timestamp, timestring): # Fake as so we read Greg format instead. We will process it later timestring = timestring.replace("%G", "%Y").replace("%V", "%W") elif '%m' in timestring: - if not '%d' in timestring: + if '%d' not in timestring: timestring += '%d' index_timestamp += '1' @@ -432,21 +469,22 @@ def get_datetime(index_timestamp, timestring): return mydate + def get_point_of_reference(unit, count, epoch=None): """ - :param unit: One of ``seconds``, ``minutes``, ``hours``, ``days``, ``weeks``, ``months``, or - ``years``. - :param unit_count: The number of ``units``. ``unit_count`` * ``unit`` will be calculated out to - the relative number of seconds. - :param epoch: An epoch timestamp used in conjunction with ``unit`` and ``unit_count`` to - establish a point of reference for calculations. + :param unit: One of ``seconds``, ``minutes``, ``hours``, ``days``, ``weeks``, + ``months``, or ``years``. + :param unit_count: The number of ``units``. ``unit_count`` * ``unit`` will be + calculated out to the relative number of seconds. + :param epoch: An epoch timestamp used in conjunction with ``unit`` and + ``unit_count`` to establish a point of reference for calculations. :type unit: str :type unit_count: int :type epoch: int - :returns: A point-of-reference timestamp in epoch + milliseconds by deriving from a ``unit`` - and a ``count``, and an optional reference timestamp, ``epoch`` + :returns: A point-of-reference timestamp in epoch + milliseconds by deriving from a + ``unit`` and a ``count``, and an optional reference timestamp, ``epoch`` :rtype: int """ if unit == 'seconds': @@ -456,13 +494,13 @@ def get_point_of_reference(unit, count, epoch=None): elif unit == 'hours': multiplier = 3600 elif unit == 'days': - multiplier = 3600*24 + multiplier = 3600 * 24 elif unit == 'weeks': - multiplier = 3600*24*7 + multiplier = 3600 * 24 * 7 elif unit == 'months': - multiplier = 3600*24*30 + multiplier = 3600 * 24 * 30 elif unit == 'years': - multiplier = 3600*24*365 + multiplier = 3600 * 24 * 365 else: raise ValueError(f'Invalid unit: {unit}.') # Use this moment as a reference point, if one is not provided. @@ -471,6 +509,7 @@ def get_point_of_reference(unit, count, epoch=None): epoch = fix_epoch(epoch) return epoch - multiplier * count + def get_unit_count_from_name(index_name, pattern): """ :param index_name: An index name @@ -494,6 +533,7 @@ def get_unit_count_from_name(index_name, pattern): else: return None + def handle_iso_week_number(mydate, timestring, index_timestamp): """ :param mydate: A Python datetime @@ -514,16 +554,20 @@ def handle_iso_week_number(mydate, timestring, index_timestamp): # Edge case 1: ISO week number is bigger than Greg week number. # Ex: year 2014, all ISO week numbers were 1 more than in Greg. - if (iso_week_str > greg_week_str or - # Edge case 2: 2010-01-01 in ISO: 2009.W53, in Greg: 2010.W00 - # For Greg converting 2009.W53 gives 2010-01-04, converting back - # to same timestring gives: 2010.W01. - datetime.strftime(mydate, timestring) != index_timestamp): + if ( + iso_week_str > greg_week_str + or + # Edge case 2: 2010-01-01 in ISO: 2009.W53, in Greg: 2010.W00 + # For Greg converting 2009.W53 gives 2010-01-04, converting back + # to same timestring gives: 2010.W01. + datetime.strftime(mydate, timestring) != index_timestamp + ): # Remove one week in this case mydate = mydate - timedelta(days=7) return mydate + def isdatemath(data): """ :param data: An expression to validate as being datemath or not @@ -540,19 +584,21 @@ def isdatemath(data): logger.debug('opener = %s, closer = %s', opener, closer) if (opener == '<' and closer != '>') or (opener != '<' and closer == '>'): raise ConfigurationError('Incomplete datemath encapsulation in "< >"') - if (opener != '<' and closer != '>'): + if opener != '<' and closer != '>': return False return True + def parse_date_pattern(name): """ - Scan and parse ``name`` for :py:func:`~.time.strftime` strings, replacing them with the - associated value when found, but otherwise returning lowercase values, as uppercase snapshot - names are not allowed. It will detect if the first character is a ``<``, which would indicate - ``name`` is going to be using Elasticsearch date math syntax, and skip accordingly. + Scan and parse ``name`` for :py:func:`~.time.strftime` strings, replacing them with + the associated value when found, but otherwise returning lowercase values, as + uppercase snapshot names are not allowed. It will detect if the first character is + a ``<``, which would indicate ``name`` is going to be using Elasticsearch date math + syntax, and skip accordingly. - The :py:func:`~.time.strftime` identifiers that Curator currently recognizes as acceptable - include: + The :py:func:`~.time.strftime` identifiers that Curator currently recognizes as + acceptable include: * ``Y``: A 4 digit year * ``y``: A 2 digit year @@ -590,12 +636,13 @@ def parse_date_pattern(name): logger.debug('Fully rendered name: %s', rendered) return rendered + def parse_datemath(client, value): """ - Validate that ``value`` looks like proper datemath. If it passes this test, then try to ship it - to Elasticsearch for real. It may yet fail this test, and if it does, it will raise a - :py:exc:`~.curator.exceptions.ConfigurationError` exception. If it passes, return the fully - parsed string. + Validate that ``value`` looks like proper datemath. If it passes this test, then + try to ship it to Elasticsearch for real. It may yet fail this test, and if it + does, it will raise a :py:exc:`~.curator.exceptions.ConfigurationError` exception. + If it passes, return the fully parsed string. :param client: A client connection object :param value: A string to check for datemath @@ -614,8 +661,8 @@ def parse_datemath(client, value): # Our pattern has 4 capture groups. # 1. Everything after the initial '<' up to the first '{', which we call ``prefix`` # 2. Everything between the outermost '{' and '}', which we call ``datemath`` - # 3. An optional inner '{' and '}' containing a date formatter and potentially a timezone. - # Not captured. + # 3. An optional inner '{' and '}' containing a date formatter and potentially a + # timezone. Not captured. # 4. Everything after the last '}' up to the closing '>' pattern = r'^<([^\{\}]*)?(\{.*(\{.*\})?\})([^\{\}]*)?>$' regex = re.compile(pattern) @@ -626,6 +673,7 @@ def parse_datemath(client, value): suffix = regex.match(value).group(4) or '' except AttributeError as exc: raise ConfigurationError( - f'Value "{value}" does not contain a valid datemath pattern.') from exc + f'Value "{value}" does not contain a valid datemath pattern.' + ) from exc return f'{prefix}{get_datemath(client, datemath)}{suffix}' diff --git a/curator/helpers/testers.py b/curator/helpers/testers.py index 7109b8cf..8399b7a5 100644 --- a/curator/helpers/testers.py +++ b/curator/helpers/testers.py @@ -1,4 +1,5 @@ """Utility functions that get things""" + import logging from voluptuous import Schema from elasticsearch8 import Elasticsearch @@ -7,12 +8,21 @@ from es_client.helpers.utils import prune_nones from curator.helpers.getters import get_repository, get_write_index from curator.exceptions import ( - ConfigurationError, MissingArgument, RepositoryException, SearchableSnapshotException) -from curator.defaults.settings import index_filtertypes, snapshot_actions, snapshot_filtertypes + ConfigurationError, + MissingArgument, + RepositoryException, + SearchableSnapshotException, +) +from curator.defaults.settings import ( + index_filtertypes, + snapshot_actions, + snapshot_filtertypes, +) from curator.validators import actions, options from curator.validators.filter_functions import validfilters from curator.helpers.utils import report_failure + def has_lifecycle_name(idx_settings): """ :param idx_settings: The settings for an index being tested @@ -26,6 +36,7 @@ def has_lifecycle_name(idx_settings): return True return False + def is_idx_partial(idx_settings): """ :param idx_settings: The settings for an index being tested @@ -41,11 +52,13 @@ def is_idx_partial(idx_settings): return True # store.snapshot.partial exists but is False -- Not a frozen tier mount return False - # store.snapshot exists, but partial isn't there -- Possibly a cold tier mount + # store.snapshot exists, but partial isn't there -- + # Possibly a cold tier mount return False raise SearchableSnapshotException('Index not a mounted searchable snapshot') raise SearchableSnapshotException('Index not a mounted searchable snapshot') + def ilm_policy_check(client, alias): """Test if alias is associated with an ILM policy @@ -71,6 +84,7 @@ def ilm_policy_check(client, alias): logger.debug('No ILM policies associated with %s', alias) return False + def repository_exists(client, repository=None): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` @@ -101,6 +115,7 @@ def repository_exists(client, repository=None): response = False return response + def rollable_alias(client, alias): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` @@ -112,8 +127,8 @@ def rollable_alias(client, alias): :type alias: str - :returns: ``True`` or ``False`` depending on whether ``alias`` is an alias that points to an - index that can be used by the ``_rollover`` API. + :returns: ``True`` or ``False`` depending on whether ``alias`` is an alias that + points to an index that can be used by the ``_rollover`` API. :rtype: bool """ logger = logging.getLogger(__name__) @@ -124,16 +139,19 @@ def rollable_alias(client, alias): return False # Response should be like: # {'there_should_be_only_one': {'aliases': {'value of "alias" here': {}}}} - # where 'there_should_be_only_one' is a single index name that ends in a number, and 'value of - # "alias" here' reflects the value of the passed parameter, except where the ``is_write_index`` - # setting makes it possible to have more than one index associated with a rollover index + # where 'there_should_be_only_one' is a single index name that ends in a number, + # and 'value of "alias" here' reflects the value of the passed parameter, except + # where the ``is_write_index`` setting makes it possible to have more than one + # index associated with a rollover index for idx in response: if 'is_write_index' in response[idx]['aliases'][alias]: if response[idx]['aliases'][alias]['is_write_index']: return True # implied ``else``: If not ``is_write_index``, it has to fit the following criteria: if len(response) > 1: - logger.error('"alias" must only reference one index, but points to %s', response) + logger.error( + '"alias" must only reference one index, but points to %s', response + ) return False index = list(response.keys())[0] rollable = False @@ -148,6 +166,7 @@ def rollable_alias(client, alias): rollable = True return rollable + def snapshot_running(client): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` @@ -170,9 +189,11 @@ def snapshot_running(client): # pylint: disable=simplifiable-if-expression return False if not status else True + def validate_actions(data): """ - Validate the ``actions`` configuration dictionary, as imported from actions.yml, for example. + Validate the ``actions`` configuration dictionary, as imported from actions.yml, + for example. :param data: The configuration dictionary @@ -203,12 +224,12 @@ def validate_actions(data): prune_nones(valid_structure['options']), options.get_schema(current_action), 'options', - loc + loc, ).result() clean_config[action_id] = { - 'action' : current_action, - 'description' : valid_structure['description'], - 'options' : clean_options, + 'action': current_action, + 'description': valid_structure['description'], + 'options': clean_options, } if current_action == 'alias': add_remove = {} @@ -218,18 +239,18 @@ def validate_actions(data): valid_structure[k]['filters'], Schema(validfilters(current_action, location=loc)), f'"{k}" filters', - f'{loc}, "filters"' + f'{loc}, "filters"', ).result() add_remove.update( { k: { - 'filters' : SchemaCheck( + 'filters': SchemaCheck( current_filters, Schema(validfilters(current_action, location=loc)), 'filters', - f'{loc}, "{k}", "filters"' - ).result() - } + f'{loc}, "{k}", "filters"', + ).result() + } } ) # Add/Remove here @@ -237,15 +258,15 @@ def validate_actions(data): elif current_action in ['cluster_routing', 'create_index', 'rollover']: # neither cluster_routing nor create_index should have filters pass - else: # Filters key only appears in non-alias actions + else: # Filters key only appears in non-alias actions valid_filters = SchemaCheck( valid_structure['filters'], Schema(validfilters(current_action, location=loc)), 'filters', - f'{loc}, "filters"' + f'{loc}, "filters"', ).result() clean_filters = validate_filters(current_action, valid_filters) - clean_config[action_id].update({'filters' : clean_filters}) + clean_config[action_id].update({'filters': clean_filters}) # This is a special case for remote reindex if current_action == 'reindex': # Check only if populated with something. @@ -254,14 +275,17 @@ def validate_actions(data): valid_structure['options']['remote_filters'], Schema(validfilters(current_action, location=loc)), 'filters', - f'{loc}, "filters"' + f'{loc}, "filters"', ).result() clean_remote_filters = validate_filters(current_action, valid_filters) - clean_config[action_id]['options'].update({'remote_filters': clean_remote_filters}) + clean_config[action_id]['options'].update( + {'remote_filters': clean_remote_filters} + ) # if we've gotten this far without any Exceptions raised, it's valid! return {'actions': clean_config} + def validate_filters(action, myfilters): """ Validate that myfilters are appropriate for the action type, e.g. no @@ -284,19 +308,21 @@ def validate_filters(action, myfilters): for fil in myfilters: if fil['filtertype'] not in filtertypes: raise ConfigurationError( - f"\"{fil['filtertype']}\" filtertype is not compatible with action \"{action}\"" + f"\"{fil['filtertype']}\" filtertype is not compatible with " + f"action \"{action}\"" ) # If we get to this point, we're still valid. Return the original list return myfilters + def verify_client_object(test): """ :param test: The variable or object to test :type test: :py:class:`~.elasticsearch.Elasticsearch` - :returns: ``True`` if ``test`` is a proper :py:class:`~.elasticsearch.Elasticsearch` client - object and raise a :py:exc:`TypeError` exception if it is not. + :returns: ``True`` if ``test`` is a proper :py:class:`~.elasticsearch.Elasticsearch` + client object and raise a :py:exc:`TypeError` exception if it is not. :rtype: bool """ logger = logging.getLogger(__name__) @@ -308,14 +334,15 @@ def verify_client_object(test): logger.error(msg) raise TypeError(msg) + def verify_index_list(test): """ :param test: The variable or object to test :type test: :py:class:`~.curator.IndexList` - :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.indexlist.IndexList` object, - else raise a :py:class:`TypeError` exception. + :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.indexlist.IndexList` + object, else raise a :py:class:`TypeError` exception. :rtype: None """ # It breaks if this import isn't local to this function: @@ -323,12 +350,14 @@ def verify_index_list(test): # 'curator.indexlist' (most likely due to a circular import) # pylint: disable=import-outside-toplevel from curator.indexlist import IndexList + logger = logging.getLogger(__name__) if not isinstance(test, IndexList): msg = f'Not a valid IndexList object. Type: {type(test)} was passed' logger.error(msg) raise TypeError(msg) + def verify_repository(client, repository=None): """ Do :py:meth:`~.elasticsearch.snapshot.verify_repository` call. If it fails, raise a @@ -365,14 +394,16 @@ def verify_repository(client, repository=None): report = f'Failed to verify all nodes have repository access: {msg}' raise RepositoryException(report) from err + def verify_snapshot_list(test): """ :param test: The variable or object to test :type test: :py:class:`~.curator.SnapshotList` - :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.snapshotlist.SnapshotList` - object, else raise a :py:class:`TypeError` exception. + :returns: ``None`` if ``test`` is a proper + :py:class:`~.curator.snapshotlist.SnapshotList` object, else raise a + :py:class:`TypeError` exception. :rtype: None """ # It breaks if this import isn't local to this function: @@ -380,6 +411,7 @@ def verify_snapshot_list(test): # 'curator.snapshotlist' (most likely due to a circular import) # pylint: disable=import-outside-toplevel from curator.snapshotlist import SnapshotList + logger = logging.getLogger(__name__) if not isinstance(test, SnapshotList): msg = f'Not a valid SnapshotList object. Type: {type(test)} was passed' diff --git a/curator/singletons.py b/curator/singletons.py index a531f7be..862746b8 100644 --- a/curator/singletons.py +++ b/curator/singletons.py @@ -1,41 +1,83 @@ """CLI module for curator_cli""" + import click from es_client.defaults import SHOW_EVERYTHING from es_client.helpers.config import ( - cli_opts, context_settings, generate_configdict, get_config, options_from_dict) + cli_opts, + context_settings, + generate_configdict, + get_config, + options_from_dict, +) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer from curator._version import __version__ from curator.cli_singletons import ( - alias, allocation, close, delete_indices, delete_snapshots, forcemerge, open_indices, replicas, - restore, rollover, snapshot, shrink + alias, + allocation, + close, + delete_indices, + delete_snapshots, + forcemerge, + open_indices, + replicas, + restore, + rollover, + snapshot, + shrink, ) from curator.cli_singletons.show import show_indices, show_snapshots click_opt_wrap = option_wrapper() -# pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals + +# pylint: disable=R0913, R0914, W0613, W0622, W0718 @click.group( - context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html')) + context_settings=context_settings(), + epilog=footer(__version__, tail='singleton-cli.html'), +) @options_from_dict(SHOW_EVERYTHING) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click.version_option(__version__, '-v', '--version', prog_name='curator_cli') @click.pass_context def curator_cli( - ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, - opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, - ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - loglevel, logfile, logformat, blacklist, dry_run + ctx, + config, + hosts, + cloud_id, + api_token, + id, + api_key, + username, + password, + bearer_auth, + opaque_id, + request_timeout, + http_compress, + verify_certs, + ca_certs, + client_cert, + client_key, + ssl_assert_hostname, + ssl_assert_fingerprint, + ssl_version, + master_only, + skip_version_test, + loglevel, + logfile, + logformat, + blacklist, + dry_run, ): """ Curator CLI (Singleton Tool) - - Run a single action from the command-line. - + + Run a single action from the command-line. + The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. - + Command-line settings will always override YAML configuration settings. """ ctx.obj = {} @@ -45,6 +87,7 @@ def curator_cli( configure_logging(ctx) generate_configdict(ctx) + # Add the subcommands curator_cli.add_command(alias) curator_cli.add_command(allocation) diff --git a/curator/validators/actions.py b/curator/validators/actions.py index 07d1a4d1..5a00e1ab 100644 --- a/curator/validators/actions.py +++ b/curator/validators/actions.py @@ -1,38 +1,43 @@ """Validate root ``actions`` and individual ``action`` Schemas""" + from voluptuous import Any, In, Schema, Optional, Required -from six import string_types from es_client.helpers.schemacheck import SchemaCheck from curator.defaults import settings + def root(): """ - Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which is a dictionary - with ``actions`` :py:class:`~.voluptuous.schema_builder.Required` to be the root key with - another dictionary as the value. + Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which + is a dictionary with ``actions`` :py:class:`~.voluptuous.schema_builder.Required` + to be the root key with another dictionary as the value. """ - return Schema({ Required('actions'): dict }) + return Schema({Required('actions'): dict}) + def valid_action(): """ - Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which is that the - value of key ``action`` is :py:class:`~.voluptuous.schema_builder.Required` to be + Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which is + that the value of key ``action`` is + :py:class:`~.voluptuous.schema_builder.Required` to be :py:class:`~.voluptuous.schema_builder.In` the value returned by :py:func:`~.curator.defaults.settings.all_actions`. """ return { Required('action'): Any( - In(settings.all_actions()), msg=f'action must be one of {settings.all_actions()}' + In(settings.all_actions()), + msg=f'action must be one of {settings.all_actions()}', ) } + def structure(data, location): """ - Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which tests ``data``, - which is ostensibly an individual action dictionary. If it is a + Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which + tests ``data``, which is ostensibly an individual action dictionary. If it is a :py:func:`~.curator.validators.actions.valid_action`, then it will :py:meth:`~.voluptuous.schema_builder.Schema.update` the base - :py:class:`~.voluptuous.schema_builder.Schema` with other options, based on the what the - value of ``data['action']`` is. + :py:class:`~.voluptuous.schema_builder.Schema` with other options, based on the + what the value of ``data['action']`` is. :param data: The configuration dictionary, or sub-dictionary, being validated :type data: dict @@ -48,12 +53,10 @@ def structure(data, location): ).result() # Build a valid schema knowing that the action has already been validated retval = valid_action() - retval.update( - {Optional('description', default='No description given'): Any(str, *string_types)} - ) + retval.update({Optional('description', default='No description given'): Any(str)}) retval.update({Optional('options', default=settings.default_options()): dict}) action = data['action'] - if action in [ 'cluster_routing', 'create_index', 'rollover']: + if action in ['cluster_routing', 'create_index', 'rollover']: # The cluster_routing, create_index, and rollover actions should not # have a 'filters' block pass diff --git a/docker_test/.env b/docker_test/.env index f874c5d1..3122d377 100644 --- a/docker_test/.env +++ b/docker_test/.env @@ -1 +1 @@ -export REMOTE_ES_SERVER="http://172.19.2.14:9201" +export REMOTE_ES_SERVER="http://192.168.64.1:9201" diff --git a/docs/Changelog.rst b/docs/Changelog.rst index b5d7cd4c..771131cd 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,23 @@ Changelog ========= +8.0.16 (6 August 2024) +---------------------- + +**Changes** + + * Update to use ``es_client==8.14.2`` + * Formatting changes and improvements + * Update CLI to get client using ``ctx.obj['configdict']`` as it's already built + by ``es_client``. + +**Bugfixes** + + * Fix improper log levels erroneously left in from debugging. Thanks to + @boutetnico in #1714 + * ``es_client`` version 8.14.2 addresses a problem where Python 3.8 is not officially supported + for use with ``voluptuous`` greater than ``0.14.2``. + 8.0.15 (10 April 2024) ---------------------- @@ -14,10 +31,10 @@ Changelog are still running Python 3.11 as cx_Freeze still does not officially support Python 3.12. * Added infrastructure to test multiple versions of Python against the code base. This requires you to run: - * ``pip install -U hatch hatchling`` -- Install prerequisites - * ``hatch run docker:create X.Y.Z`` -- where ``X.Y.Z`` is an ES version on Docker Hub - * ``hatch run test:pytest`` -- Run the test suite for each supported version of Python - * ``hatch run docker:destroy`` -- Cleanup the Docker containers created in ``docker:create`` + * ``pip install -U hatch hatchling`` -- Install prerequisites + * ``hatch run docker:create X.Y.Z`` -- where ``X.Y.Z`` is an ES version on Docker Hub + * ``hatch run test:pytest`` -- Run the test suite for each supported version of Python + * ``hatch run docker:destroy`` -- Cleanup the Docker containers created in ``docker:create`` **Bugfix** diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index 8e50a310..0c52bff1 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,9 +1,9 @@ -:curator_version: 8.0.15 +:curator_version: 8.0.16 :curator_major: 8 :curator_doc_tree: 8.0 -:es_py_version: 8.13.0 -:es_doc_tree: 8.13 -:stack_doc_tree: 8.13 +:es_py_version: 8.14.0 +:es_doc_tree: 8.14 +:stack_doc_tree: 8.14 :pybuild_ver: 3.11.9 :copyright_years: 2011-2024 :ref: http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree} diff --git a/docs/conf.py b/docs/conf.py index 8cc8bdb6..741d9aaf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,9 +71,9 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] intersphinx_mapping = { - 'python': ('https://docs.python.org/3.11', None), - 'es_client': ('https://es-client.readthedocs.io/en/v8.13.1', None), - 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.13.0', None), + 'python': ('https://docs.python.org/3.11', None), + 'es_client': ('https://es-client.readthedocs.io/en/v8.14.2', None), + 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.14.0', None), 'voluptuous': ('http://alecthomas.github.io/voluptuous/docs/_build/html', None), 'click': ('https://click.palletsprojects.com/en/8.1.x', None), } diff --git a/docs/helpers.rst b/docs/helpers.rst index 6c2acde0..f7d2a945 100644 --- a/docs/helpers.rst +++ b/docs/helpers.rst @@ -55,8 +55,6 @@ Getters .. autofunction:: get_data_tiers -.. autofunction:: get_frozen_prefix - .. autofunction:: get_indices .. autofunction:: get_repository diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..82beb7a7 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +plugins = returns.contrib.mypy.returns_plugin diff --git a/pyproject.toml b/pyproject.toml index bf3ac20c..8a87df75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ keywords = [ 'index-expiry' ] dependencies = [ - "es_client==8.13.1" + "es_client==8.14.2" ] [project.optional-dependencies] @@ -84,6 +84,14 @@ create = "docker_test/scripts/create.sh {args}" destroy = "docker_test/scripts/destroy.sh" ### Lint environment +[tool.hatch.envs.lint] +detached = true +dependencies = [ + 'black>=23.1.0', + 'mypy>=1.0.0', + 'ruff>=0.0.243', +] + [tool.hatch.envs.lint.scripts] run-pyright = "pyright {args:.}" run-black = "black --quiet --check --diff {args:.}" @@ -93,6 +101,15 @@ python = ["run-pyright", "run-black", "run-ruff"] templates = ["run-curlylint"] all = ["python", "templates"] +[tool.pylint.format] +max-line-length = "88" + +[tool.black] +target-version = ['py38'] +line-length = 88 +skip-string-normalization = true +include = '\.pyi?$' + ### Test environment [tool.hatch.envs.test] platforms = ["linux", "macos"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..04934b6f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +log_format = %(asctime)s %(levelname)-9s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s diff --git a/tests/integration/test_reindex.py b/tests/integration/test_reindex.py index ed698af8..d3db6621 100644 --- a/tests/integration/test_reindex.py +++ b/tests/integration/test_reindex.py @@ -1,9 +1,11 @@ """Test reindex action functionality""" + # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long import os from unittest.case import SkipTest import pytest -from es_client.builder import ClientArgs, Builder +from dotmap import DotMap # type: ignore +from es_client.builder import Builder from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars @@ -14,8 +16,10 @@ WAIT_INTERVAL = 1 MAX_WAIT = 3 + class TestActionFileReindex(CuratorTestCase): """Test file-based reindex operations""" + def test_reindex_manual(self): """Test that manual reindex results in proper count of documents""" source = 'my_source' @@ -24,9 +28,13 @@ def test_reindex_manual(self): self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), + ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] + def test_reindex_selected(self): """Reindex selected indices""" source = 'my_source' @@ -35,9 +43,13 @@ def test_reindex_selected(self): self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest), + ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] + def test_reindex_empty_list(self): """ This test raises : @@ -49,9 +61,13 @@ def test_reindex_empty_list(self): dest = 'my_dest' expected = [] self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), + ) self.invoke_runner() assert expected == get_indices(self.client) + def test_reindex_selected_many_to_one(self): """Test reindexing many indices to one destination""" source1 = 'my_source1' @@ -62,16 +78,22 @@ def test_reindex_selected_many_to_one(self): self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: - self.client.create(index=source2, id=i, document={"doc" + i :'TEST DOCUMENT'}) + self.client.create( + index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} + ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.client.indices.refresh(index=source2) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest), + ) self.invoke_runner() self.client.indices.refresh(index=dest) assert expected == self.client.count(index=dest)['count'] + def test_reindex_selected_empty_list_fail(self): """Ensure an empty list results in an exit code 1""" source1 = 'my_source1' @@ -81,14 +103,20 @@ def test_reindex_selected_empty_list_fail(self): self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: - self.client.create(index=source2, id=i, document={"doc" + i :'TEST DOCUMENT'}) + self.client.create( + index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} + ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex_empty_list.format('false', WAIT_INTERVAL, MAX_WAIT, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex_empty_list.format('false', WAIT_INTERVAL, MAX_WAIT, dest), + ) self.invoke_runner() assert 1 == self.result.exit_code + def test_reindex_selected_empty_list_pass(self): """Ensure an empty list results in an exit code 0""" source1 = 'my_source1' @@ -98,12 +126,17 @@ def test_reindex_selected_empty_list_pass(self): self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: - self.client.create(index=source2, id=i, document={"doc" + i :'TEST DOCUMENT'}) + self.client.create( + index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} + ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex_empty_list.format('true', WAIT_INTERVAL, MAX_WAIT, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex_empty_list.format('true', WAIT_INTERVAL, MAX_WAIT, dest), + ) self.invoke_runner() assert 0 == self.result.exit_code @@ -118,36 +151,37 @@ def test_reindex_from_remote(self): expected = 6 # Build remote client try: - remote_args = ClientArgs() - remote_args.hosts = RHOST - remote_config = {'elasticsearch': {'client': remote_args.asdict()}} - builder = Builder(configdict=remote_config, version_min=(5,0,0)) + remote_args = DotMap({'hosts': RHOST}) + remote_config = {'elasticsearch': {'client': remote_args.toDict()}} + builder = Builder(configdict=remote_config) + builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: - raise SkipTest(f'Unable to connect to host at {RHOST}') from exc + raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): - rclient.create(index=rindex, id=str(counter+1), document={"doc" + str(i) :'TEST DOCUMENT'}) + rclient.create( + index=rindex, + id=str(counter + 1), + document={"doc" + str(i): 'TEST DOCUMENT'}, + ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.remote_reindex.format( - WAIT_INTERVAL, - diff_wait, - RHOST, - 'REINDEX_SELECTION', - dest, - prefix - ) + self.write_config( + self.args['actionfile'], + testvars.remote_reindex.format( + WAIT_INTERVAL, diff_wait, RHOST, 'REINDEX_SELECTION', dest, prefix + ), ) self.invoke_runner() # Do our own cleanup here. @@ -164,36 +198,37 @@ def test_reindex_migrate_from_remote(self): expected = 3 # Build remote client try: - remote_args = ClientArgs() - remote_args.hosts = RHOST - remote_config = {'elasticsearch': {'client': remote_args.asdict()}} - builder = Builder(configdict=remote_config, version_min=(5,0,0)) + remote_args = DotMap({'hosts': RHOST}) + remote_config = {'elasticsearch': {'client': remote_args.toDict()}} + builder = Builder(configdict=remote_config) + builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: - raise SkipTest(f'Unable to connect to host at {RHOST}') from exc + raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): - rclient.create(index=rindex, id=str(counter+1), document={"doc" + str(i) :'TEST DOCUMENT'}) + rclient.create( + index=rindex, + id=str(counter + 1), + document={"doc" + str(i): 'TEST DOCUMENT'}, + ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.remote_reindex.format( - WAIT_INTERVAL, - MAX_WAIT, - RHOST, - 'REINDEX_SELECTION', - dest, - prefix - ) + self.write_config( + self.args['actionfile'], + testvars.remote_reindex.format( + WAIT_INTERVAL, MAX_WAIT, RHOST, 'REINDEX_SELECTION', dest, prefix + ), ) self.invoke_runner() # Do our own cleanup here. @@ -215,29 +250,35 @@ def test_reindex_migrate_from_remote_with_pre_suf_fixes(self): msfx = '-fix' # Build remote client try: - remote_args = ClientArgs() - remote_args.hosts = RHOST - remote_config = {'elasticsearch': {'client': remote_args.asdict()}} - builder = Builder(configdict=remote_config, version_min=(5,0,0)) + remote_args = DotMap({'hosts': RHOST}) + remote_config = {'elasticsearch': {'client': remote_args.toDict()}} + builder = Builder(configdict=remote_config) + builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: - raise SkipTest(f'Unable to connect to host at {RHOST}') from exc + raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): - rclient.create(index=rindex, id=str(counter+1), document={"doc" + str(i) :'TEST DOCUMENT'}) + rclient.create( + index=rindex, + id=str(counter + 1), + document={"doc" + str(i): 'TEST DOCUMENT'}, + ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.migration_reindex.format( + self.write_config( + self.args['actionfile'], + testvars.migration_reindex.format( WAIT_INTERVAL, MAX_WAIT, mpfx, @@ -245,8 +286,8 @@ def test_reindex_migrate_from_remote_with_pre_suf_fixes(self): RHOST, 'REINDEX_SELECTION', dest, - prefix - ) + prefix, + ), ) self.invoke_runner() # Do our own cleanup here. @@ -254,27 +295,27 @@ def test_reindex_migrate_from_remote_with_pre_suf_fixes(self): # And now the neat trick of verifying that the reindex worked to both # indices, and they preserved their names assert expected == self.client.count(index=f'{mpfx}{source1}{msfx}')['count'] + def test_reindex_from_remote_no_connection(self): """Ensure that the inability to connect to the remote cluster fails""" dest = 'my_dest' bad_remote = 'http://127.0.0.1:9601' expected = 1 self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.remote_reindex.format( - WAIT_INTERVAL, - MAX_WAIT, - bad_remote, - 'REINDEX_SELECTION', - dest, - 'my_' - ) + self.write_config( + self.args['actionfile'], + testvars.remote_reindex.format( + WAIT_INTERVAL, MAX_WAIT, bad_remote, 'REINDEX_SELECTION', dest, 'my_' + ), ) self.invoke_runner() assert expected == self.result.exit_code @pytest.mark.skipif((RHOST == UNDEF), reason='REMOTE_ES_SERVER is not defined') def test_reindex_from_remote_no_indices(self): - """Test that attempting to reindex remotely with an empty list exits with a fail""" + """ + Test that attempting to reindex remotely with an empty list exits with a fail + """ source1 = 'wrong1' source2 = 'wrong2' prefix = 'my_' @@ -282,52 +323,60 @@ def test_reindex_from_remote_no_indices(self): expected = 1 # Build remote client try: - remote_args = ClientArgs() - remote_args.hosts = RHOST - remote_config = {'elasticsearch': {'client': remote_args.asdict()}} - builder = Builder(configdict=remote_config, version_min=(5,0,0)) + remote_args = DotMap({'hosts': RHOST}) + remote_config = {'elasticsearch': {'client': remote_args.toDict()}} + builder = Builder(configdict=remote_config) + builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: - raise SkipTest(f'Unable to connect to host at {RHOST}') from exc + raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 # Force remove my_source1 and my_source2 to prevent false positives - rclient.indices.delete(index=f"{'my_source1'},{'my_source2'}", ignore_unavailable=True) + rclient.indices.delete( + index=f"{'my_source1'},{'my_source2'}", ignore_unavailable=True + ) rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): - rclient.create(index=rindex, id=str(counter+1), document={"doc" + str(i) :'TEST DOCUMENT'}) + rclient.create( + index=rindex, + id=str(counter + 1), + document={"doc" + str(i): 'TEST DOCUMENT'}, + ) counter += 1 rclient.indices.flush(index=rindex, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.remote_reindex.format( - WAIT_INTERVAL, - MAX_WAIT, - f'{RHOST}', - 'REINDEX_SELECTION', - dest, - prefix - ) + self.write_config( + self.args['actionfile'], + testvars.remote_reindex.format( + WAIT_INTERVAL, MAX_WAIT, f'{RHOST}', 'REINDEX_SELECTION', dest, prefix + ), ) self.invoke_runner() # Do our own cleanup here. rclient.indices.delete(index=f'{source1},{source2}') assert expected == self.result.exit_code + def test_reindex_into_alias(self): """Ensure that reindexing into an alias works as expected""" source = 'my_source' dest = 'my_dest' expected = 3 - alias_dict = {dest : {}} + alias_dict = {dest: {}} self.client.indices.create(index='dummy', aliases=alias_dict) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), + ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] + def test_reindex_manual_date_math(self): """Ensure date math is functional with reindex calls""" source = '' @@ -337,22 +386,29 @@ def test_reindex_manual_date_math(self): self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), + ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] + def test_reindex_bad_mapping(self): """This test addresses GitHub issue #1260""" source = 'my_source' dest = 'my_dest' expected = 1 - settings = { "number_of_shards": 1, "number_of_replicas": 0} - mappings1 = { "properties": { "doc1": { "type": "keyword" }}} - mappings2 = { "properties": { "doc1": { "type": "integer" }}} + settings = {"number_of_shards": 1, "number_of_replicas": 0} + mappings1 = {"properties": {"doc1": {"type": "keyword"}}} + mappings2 = {"properties": {"doc1": {"type": "integer"}}} self.client.indices.create(index=source, settings=settings, mappings=mappings1) self.add_docs(source) # Create the dest index with a different mapping. self.client.indices.create(index=dest, settings=settings, mappings=mappings2) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - self.write_config(self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest)) + self.write_config( + self.args['actionfile'], + testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), + ) self.invoke_runner() assert expected == self.result.exit_code diff --git a/tests/unit/test_helpers_waiters.py b/tests/unit/test_helpers_waiters.py index ea7aebde..82ab3c62 100644 --- a/tests/unit/test_helpers_waiters.py +++ b/tests/unit/test_helpers_waiters.py @@ -1,28 +1,60 @@ """Unit tests for utils""" + from unittest import TestCase -import pytest from unittest.mock import Mock -from curator.exceptions import ActionTimeout, ConfigurationError, CuratorException, MissingArgument +import pytest +from curator.exceptions import ( + ActionTimeout, + ConfigurationError, + CuratorException, + MissingArgument, +) from curator.helpers.waiters import ( - health_check, restore_check, snapshot_check,task_check, wait_for_it) + health_check, + restore_check, + snapshot_check, + task_check, + wait_for_it, +) FAKE_FAIL = Exception('Simulated Failure') + class TestHealthCheck(TestCase): """TestHealthCheck Test helpers.waiters.health_check functionality """ + # pylint: disable=line-too-long - CLUSTER_HEALTH = {"cluster_name": "unit_test", "status": "green", "timed_out": False, "number_of_nodes": 7, "number_of_data_nodes": 3, "active_primary_shards": 235, "active_shards": 471, "relocating_shards": 0, "initializing_shards": 0, "unassigned_shards": 0, "delayed_unassigned_shards": 0, "number_of_pending_tasks": 0, "task_max_waiting_in_queue_millis": 0, "active_shards_percent_as_number": 100} + CLUSTER_HEALTH = { + "cluster_name": "unit_test", + "status": "green", + "timed_out": False, + "number_of_nodes": 7, + "number_of_data_nodes": 3, + "active_primary_shards": 235, + "active_shards": 471, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100, + } + def test_no_kwargs(self): """test_no_kwargs Should raise a ``MissingArgument`` exception when no keyword args are passed. """ client = Mock() - with pytest.raises(MissingArgument, match=r'Must provide at least one keyword argument'): + with pytest.raises( + MissingArgument, match=r'Must provide at least one keyword argument' + ): health_check(client) + def test_key_value_match(self): """test_key_value_match @@ -31,31 +63,38 @@ def test_key_value_match(self): client = Mock() client.cluster.health.return_value = self.CLUSTER_HEALTH assert health_check(client, status='green') + def test_key_value_no_match(self): """test_key_value_no_match - Should return ``False`` when matching keyword args are passed, but no matches are found. + Should return ``False`` when matching keyword args are passed, but no matches + are found. """ client = Mock() client.cluster.health.return_value = self.CLUSTER_HEALTH assert not health_check(client, status='red') + def test_key_not_found(self): """test_key_not_found - Should raise ``ConfigurationError`` when keyword args are passed, but keys match. + Should raise ``ConfigurationError`` when keyword args are passed, but keys + match. """ client = Mock() client.cluster.health.return_value = self.CLUSTER_HEALTH with pytest.raises(ConfigurationError, match=r'not in cluster health output'): health_check(client, foo='bar') + class TestRestoreCheck(TestCase): """TestRestoreCheck Test helpers.waiters.restore_check functionality """ + SNAP_NAME = 'snap_name' - NAMED_INDICES = [ "index-2015.01.01", "index-2015.02.01" ] + NAMED_INDICES = ["index-2015.01.01", "index-2015.02.01"] + def test_fail_to_get_recovery(self): """test_fail_to_get_recovery @@ -63,8 +102,11 @@ def test_fail_to_get_recovery(self): """ client = Mock() client.indices.recovery.side_effect = FAKE_FAIL - with pytest.raises(CuratorException, match=r'Unable to obtain recovery information'): + with pytest.raises( + CuratorException, match=r'Unable to obtain recovery information' + ): restore_check(client, []) + def test_incomplete_recovery(self): """test_incomplete_recovery @@ -72,8 +114,12 @@ def test_incomplete_recovery(self): """ client = Mock() # :pylint disable=line-too-long - client.indices.recovery.return_value = {'index-2015.01.01': {'shards' : [{'stage':'INDEX'}]}, 'index-2015.02.01': {'shards' : [{'stage':'INDEX'}]}} + client.indices.recovery.return_value = { + 'index-2015.01.01': {'shards': [{'stage': 'INDEX'}]}, + 'index-2015.02.01': {'shards': [{'stage': 'INDEX'}]}, + } assert not restore_check(client, self.NAMED_INDICES) + def test_completed_recovery(self): """test_completed_recovery @@ -81,8 +127,12 @@ def test_completed_recovery(self): """ client = Mock() # :pylint disable=line-too-long - client.indices.recovery.return_value = {'index-2015.01.01': {'shards' : [{'stage':'DONE'}]}, 'index-2015.02.01': {'shards' : [{'stage':'DONE'}]}} + client.indices.recovery.return_value = { + 'index-2015.01.01': {'shards': [{'stage': 'DONE'}]}, + 'index-2015.02.01': {'shards': [{'stage': 'DONE'}]}, + } assert restore_check(client, self.NAMED_INDICES) + def test_empty_recovery(self): """test_empty_recovery @@ -92,14 +142,17 @@ def test_empty_recovery(self): client.indices.recovery.return_value = {} assert not restore_check(client, self.NAMED_INDICES) + class TestSnapshotCheck(TestCase): """TestSnapshotCheck Test helpers.waiters.snapshot_check functionality """ + # :pylint disable=line-too-long SNAP_NAME = 'snap_name' - NAMED_INDICES = [ "index-2015.01.01", "index-2015.02.01" ] + NAMED_INDICES = ["index-2015.01.01", "index-2015.02.01"] + def test_fail_to_get_snapshot(self): """test_fail_to_get_snapshot @@ -108,65 +161,116 @@ def test_fail_to_get_snapshot(self): client = Mock() client.snapshot.get.side_effect = FAKE_FAIL self.assertRaises(CuratorException, snapshot_check, client) + def test_in_progress(self): """test_in_progress Should return ``False`` when state is ``IN_PROGRESS``. """ client = Mock() - test_val = {'snapshots': - [{'state': 'IN_PROGRESS', 'snapshot': self.SNAP_NAME, 'indices': self.NAMED_INDICES}]} + test_val = { + 'snapshots': [ + { + 'state': 'IN_PROGRESS', + 'snapshot': self.SNAP_NAME, + 'indices': self.NAMED_INDICES, + } + ] + } client.snapshot.get.return_value = test_val assert not snapshot_check(client, repository='foo', snapshot=self.SNAP_NAME) + def test_success(self): """test_success Should return ``True`` when state is ``SUCCESS``. """ client = Mock() - test_val = {'snapshots': - [{'state': 'SUCCESS', 'snapshot': self.SNAP_NAME, 'indices': self.NAMED_INDICES}]} + test_val = { + 'snapshots': [ + { + 'state': 'SUCCESS', + 'snapshot': self.SNAP_NAME, + 'indices': self.NAMED_INDICES, + } + ] + } client.snapshot.get.return_value = test_val assert snapshot_check(client, repository='foo', snapshot=self.SNAP_NAME) + def test_partial(self): """test_partial Should return ``True`` when state is ``PARTIAL``. """ client = Mock() - test_val = {'snapshots': - [{'state': 'PARTIAL', 'snapshot': self.SNAP_NAME, 'indices': self.NAMED_INDICES}]} + test_val = { + 'snapshots': [ + { + 'state': 'PARTIAL', + 'snapshot': self.SNAP_NAME, + 'indices': self.NAMED_INDICES, + } + ] + } client.snapshot.get.return_value = test_val assert snapshot_check(client, repository='foo', snapshot=self.SNAP_NAME) + def test_failed(self): """test_failed Should return ``True`` when state is ``FAILED``. """ client = Mock() - test_val = {'snapshots': - [{'state': 'FAILED', 'snapshot': self.SNAP_NAME, 'indices': self.NAMED_INDICES}]} + test_val = { + 'snapshots': [ + { + 'state': 'FAILED', + 'snapshot': self.SNAP_NAME, + 'indices': self.NAMED_INDICES, + } + ] + } client.snapshot.get.return_value = test_val assert snapshot_check(client, repository='foo', snapshot=self.SNAP_NAME) + def test_other(self): """test_other - Should return ``True`` when state is anything other than ``IN_PROGRESS`` or the above. + Should return ``True`` when state is anything other than ``IN_PROGRESS`` or the + above. """ client = Mock() - test_val = {'snapshots': - [{'state': 'SOMETHINGELSE', 'snapshot': self.SNAP_NAME, 'indices': self.NAMED_INDICES}]} + test_val = { + 'snapshots': [ + { + 'state': 'SOMETHINGELSE', + 'snapshot': self.SNAP_NAME, + 'indices': self.NAMED_INDICES, + } + ] + } client.snapshot.get.return_value = test_val assert snapshot_check(client, repository='foo', snapshot=self.SNAP_NAME) + class TestTaskCheck(TestCase): """TestTaskCheck Test helpers.waiters.task_check functionality """ + # pylint: disable=line-too-long - PROTO_TASK = {'node': 'I0ekFjMhSPCQz7FUs1zJOg', 'description': 'UNIT TEST', 'running_time_in_nanos': 1637039537721, 'action': 'indices:data/write/reindex', 'id': 54510686, 'start_time_in_millis': 1489695981997} + PROTO_TASK = { + 'node': 'I0ekFjMhSPCQz7FUs1zJOg', + 'description': 'UNIT TEST', + 'running_time_in_nanos': 1637039537721, + 'action': 'indices:data/write/reindex', + 'id': 54510686, + 'start_time_in_millis': 1489695981997, + } GENERIC_TASK = {'task': 'I0ekFjMhSPCQz7FUs1zJOg:54510686'} + def test_bad_task_id(self): """test_bad_task_id @@ -174,32 +278,46 @@ def test_bad_task_id(self): """ client = Mock() client.tasks.get.side_effect = FAKE_FAIL - with pytest.raises(CuratorException, match=r'Unable to obtain task information for task'): + with pytest.raises( + CuratorException, match=r'Unable to obtain task information for task' + ): task_check(client, 'foo') + def test_incomplete_task(self): """test_incomplete_task Should return ``False`` if task is incomplete """ client = Mock() - test_task = {'completed': False, 'task': self.PROTO_TASK, 'response': {'failures': []}} + test_task = { + 'completed': False, + 'task': self.PROTO_TASK, + 'response': {'failures': []}, + } client.tasks.get.return_value = test_task assert not task_check(client, task_id=self.GENERIC_TASK['task']) + def test_complete_task(self): """test_complete_task Should return ``True`` if task is complete """ client = Mock() - test_task = {'completed': True, 'task': self.PROTO_TASK, 'response': {'failures': []}} + test_task = { + 'completed': True, + 'task': self.PROTO_TASK, + 'response': {'failures': []}, + } client.tasks.get.return_value = test_task assert task_check(client, task_id=self.GENERIC_TASK['task']) + class TestWaitForIt(TestCase): """TestWaitForIt Test helpers.waiters.wait_for_it functionality """ + # pylint: disable=line-too-long def test_bad_action(self): """test_bad_action @@ -210,42 +328,59 @@ def test_bad_action(self): # self.assertRaises(ConfigurationError, wait_for_it, client, 'foo') with pytest.raises(ConfigurationError, match=r'"action" must be one of'): wait_for_it(client, 'foo') + def test_reindex_action_no_task_id(self): """test_reindex_action_no_task_id - Should raise a ``MissingArgument`` exception if ``task_id`` is missing for ``reindex`` + Should raise a ``MissingArgument`` exception if ``task_id`` is missing for + ``reindex`` """ client = Mock() # self.assertRaises(MissingArgument, wait_for_it, client, 'reindex') with pytest.raises(MissingArgument, match=r'A task_id must accompany "action"'): wait_for_it(client, 'reindex') + def test_snapshot_action_no_snapshot(self): """test_snapshot_action_no_snapshot - Should raise a ``MissingArgument`` exception if ``snapshot`` is missing for ``snapshot`` + Should raise a ``MissingArgument`` exception if ``snapshot`` is missing for + ``snapshot`` """ client = Mock() - # self.assertRaises(MissingArgument, wait_for_it, client, 'snapshot', repository='foo') - with pytest.raises(MissingArgument, match=r'A snapshot and repository must accompany "action"'): + # self.assertRaises(MissingArgument, wait_for_it, client, + # 'snapshot', repository='foo') + with pytest.raises( + MissingArgument, match=r'A snapshot and repository must accompany "action"' + ): wait_for_it(client, 'snapshot', repository='foo') + def test_snapshot_action_no_repository(self): """test_snapshot_action_no_repository - Should raise a ``MissingArgument`` exception if ``repository`` is missing for ``snapshot`` + Should raise a ``MissingArgument`` exception if ``repository`` is missing for + ``snapshot`` """ client = Mock() - # self.assertRaises(MissingArgument, wait_for_it, client, 'snapshot', snapshot='foo') - with pytest.raises(MissingArgument, match=r'A snapshot and repository must accompany "action"'): + # self.assertRaises(MissingArgument, wait_for_it, client, + # 'snapshot', snapshot='foo') + with pytest.raises( + MissingArgument, match=r'A snapshot and repository must accompany "action"' + ): wait_for_it(client, 'snapshot', snapshot='foo') + def test_restore_action_no_indexlist(self): """test_restore_action_no_indexlist - Should raise a ``MissingArgument`` exception if ``index_list`` is missing for ``restore`` + Should raise a ``MissingArgument`` exception if ``index_list`` is missing for + ``restore`` """ client = Mock() # self.assertRaises(MissingArgument, wait_for_it, client, 'restore') - with pytest.raises(MissingArgument, match=r'An index_list must accompany "action"'): + with pytest.raises( + MissingArgument, match=r'An index_list must accompany "action"' + ): wait_for_it(client, 'restore') + def test_reindex_action_bad_task_id(self): """test_reindex_action_bad_task_id @@ -254,18 +389,24 @@ def test_reindex_action_bad_task_id(self): This is kind of a fake fail, even in the code. """ client = Mock() - client.tasks.get.return_value = {'a':'b'} + client.tasks.get.return_value = {'a': 'b'} client.tasks.get.side_effect = FAKE_FAIL - # self.assertRaises(CuratorException, wait_for_it, client, 'reindex', task_id='foo') + # self.assertRaises(CuratorException, wait_for_it, + # client, 'reindex', task_id='foo') with pytest.raises(CuratorException, match=r'Unable to find task_id'): wait_for_it(client, 'reindex', task_id='foo') + def test_reached_max_wait(self): """test_reached_max_wait - Should raise a ``ActionTimeout`` exception if we've waited past the defined timeout period + Should raise a ``ActionTimeout`` exception if we've waited past the defined + timeout period """ client = Mock() - client.cluster.health.return_value = {'status':'red'} - # self.assertRaises(ActionTimeout, wait_for_it, client, 'replicas', wait_interval=1, max_wait=1) - with pytest.raises(ActionTimeout, match=r'failed to complete in the max_wait period'): + client.cluster.health.return_value = {'status': 'red'} + # self.assertRaises(ActionTimeout, wait_for_it, client, 'replicas', + # wait_interval=1, max_wait=1) + with pytest.raises( + ActionTimeout, match=r'failed to complete in the max_wait period' + ): wait_for_it(client, 'replicas', wait_interval=1, max_wait=1)