diff --git a/curator/_version.py b/curator/_version.py index 3be392bd..50efedc0 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.1' +__version__ = '8.0.2' diff --git a/curator/cli.py b/curator/cli.py index b8db8a06..15f4e08d 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -2,8 +2,8 @@ import sys import logging import click -from es_client.builder import ClientArgs, OtherArgs, Builder -from es_client.helpers.utils import get_yaml, check_config, prune_nones +from es_client.builder import ClientArgs, OtherArgs +from es_client.helpers.utils import get_yaml, check_config, prune_nones, verify_url_schema from curator.actions import ( Alias, Allocation, Close, ClusterRouting, CreateIndex, DeleteIndices, ForceMerge, IndexSettings, Open, Reindex, Replicas, Rollover, Shrink, Snapshot, DeleteSnapshots, Restore @@ -12,10 +12,11 @@ from curator.config_utils import check_logging_config, password_filter, set_logging from curator.defaults import settings from curator.exceptions import NoIndices, NoSnapshots -from curator.helpers.getters import get_write_index -from curator.helpers.testers import validate_actions +from curator.helpers.getters import get_client +from curator.helpers.testers import ilm_policy_check, validate_actions from curator.indexlist import IndexList from curator.snapshotlist import SnapshotList +from curator.cli_singletons.utils import get_width from curator._version import __version__ CLASS_MAP = { @@ -37,6 +38,50 @@ 'shrink' : Shrink, } +def override_logging(config, loglevel, logfile, logformat): + """Get logging config and override from command-line options + + :param config: The configuration from file + :param loglevel: The log level + :param logfile: The log file to write + :param logformat: Which log format to use + + :type config: dict + :type loglevel: str + :type logfile: str + :type logformat: str + + :returns: Log configuration ready for validation + :rtype: dict + """ + # Check for log settings from config file + init_logcfg = check_logging_config(config) + + # Override anything with options from the command-line + if loglevel: + init_logcfg['loglevel'] = loglevel + if logfile: + init_logcfg['logfile'] = logfile + if logformat: + init_logcfg['logformat'] = logformat + return init_logcfg + +def cli_hostslist(hosts): + """ + :param hosts: One or more hosts. + :type hosts: str or list + + :returns: A list of hosts that came in from the command-line, or ``None`` + :rtype: list or ``None`` + """ + hostslist = [] + if hosts: + for host in list(hosts): + hostslist.append(verify_url_schema(host)) + else: + hostslist = None + return hostslist + def process_action(client, config, **kwargs): """ Do the ``action`` in ``config``, using the associated options and ``kwargs``, if any. @@ -106,38 +151,21 @@ def process_action(client, config, **kwargs): logger.debug('Doing the action here.') action_obj.do_action() -def run(config, action_file, dry_run=False): +def run(client_args, other_args, action_file, dry_run=False): """ Called by :py:func:`cli` to execute what was collected at the command-line + + :param client_args: The ClientArgs arguments object + :param other_args: The OtherArgs arguments object + :param action_file: The action configuration file + :param dry_run: Do not perform any changes + + :type client_args: :py:class:`~.es_client.ClientArgs` + :type other_args: :py:class:`~.es_client.OtherArgs` + :type action_file: str + :type dry_run: bool """ - # """Process yaml_file and return a valid client configuration""" - config_dict = get_yaml(config) - if config_dict is None: - click.echo('Empty configuration file provided. Using defaults') - config_dict = {} - elif not isinstance(config_dict, dict): - raise ConfigurationError('Configuration file not converted to dictionary. Check YAML configuration.') - set_logging(check_logging_config(config_dict)) - # set_logging({'loglevel':'DEBUG','blacklist':[]}) logger = logging.getLogger(__name__) - if not isinstance(config_dict, dict): - config_dict = {} - logger.warning( - 'Provided config file %s was unable to be properly read, or is empty. ' - 'Using empty dictionary (assuming defaults)', config) - logger.debug('config_dict = %s', config_dict) - client_args = ClientArgs() - other_args = OtherArgs() - if config: - raw_config = check_config(config_dict) - logger.debug('raw_config = %s', raw_config) - try: - client_args.update_settings(raw_config['client']) - # pylint: disable=broad-except - except Exception as exc: - click.echo(f'EXCEPTION = {exc}') - sys.exit(1) - other_args.update_settings(raw_config['other_settings']) logger.debug('Client and logging configuration options validated.') @@ -189,22 +217,19 @@ def run(config, action_file, dry_run=False): # Create a client object for each action... logger.info('Creating client object and testing connection') - # Build a "final_config" that reflects CLI args overriding anything from a config_file - final_config = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } - builder = Builder(configdict=final_config) - try: - builder.connect() - except Exception as exc: - click.echo(f'Exception encountered: {exc}') - raise ClientException from exc + client = get_client(configdict={ + 'elasticsearch': { + 'client': prune_nones(client_args.asdict()), + 'other_settings': prune_nones(other_args.asdict()) + } + }) + except ClientException as exc: + # No matter where logging is set to go, make sure we dump these messages to the CLI + click.echo('Unable to establish client connection to Elasticsearch!') + click.echo(f'Exception: {exc}') + sys.exit(1) - client = builder.client ### Filter ILM indices unless expressly permitted if allow_ilm: logger.warning('allow_ilm_indices: true') @@ -212,10 +237,8 @@ def run(config, action_file, dry_run=False): if not allow_ilm and action not in settings.snapshot_actions(): if actions_config[idx]['action'] == 'rollover': alias = actions_config[idx]['options']['name'] - write_index = get_write_index(client, alias) try: - idx_settings = client.indices.get_settings(index=write_index) - if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: + if ilm_policy_check(client, alias): logger.info('Alias %s is associated with ILM policy.', alias) logger.info('Skipping action %s because allow_ilm_indices is false.', idx) continue @@ -235,7 +258,7 @@ def run(config, action_file, dry_run=False): process_action(client, actions_config[idx], **kwargs) # pylint: disable=broad-except except Exception as err: - if isinstance(err, NoIndices) or isinstance(err, NoSnapshots): + if isinstance(err, (NoIndices, NoSnapshots)): if ignore_empty_list: logger.info('Skipping action "%s" due to empty list: %s', action, type(err)) else: @@ -250,19 +273,106 @@ def run(config, action_file, dry_run=False): logger.info('Action ID: %s, "%s" completed.', idx, action) logger.info('Job completed.') -@click.command() -@click.option( - '--config', - help="Path to configuration file. Default: ~/.curator/curator.yml", - type=click.Path(exists=True), default=settings.config_file() -) +@click.command(context_settings=get_width()) +@click.option('--config', help='Path to configuration file.', type=click.Path(exists=True), default=settings.config_file()) +@click.option('--hosts', help='Elasticsearch URL to connect to', multiple=True) +@click.option('--cloud_id', help='Shorthand to connect to Elastic Cloud instance') +@click.option('--id', help='API Key "id" value', type=str) +@click.option('--api_key', help='API Key "api_key" value', type=str) +@click.option('--username', help='Username used to create "basic_auth" tuple') +@click.option('--password', help='Password used to create "basic_auth" tuple') +@click.option('--bearer_auth', type=str) +@click.option('--opaque_id', type=str) +@click.option('--request_timeout', help='Request timeout in seconds', type=float) +@click.option('--http_compress', help='Enable HTTP compression', is_flag=True, default=None) +@click.option('--verify_certs', help='Verify SSL/TLS certificate(s)', is_flag=True, default=None) +@click.option('--ca_certs', help='Path to CA certificate file or directory') +@click.option('--client_cert', help='Path to client certificate file') +@click.option('--client_key', help='Path to client certificate key') +@click.option('--ssl_assert_hostname', help='Hostname or IP address to verify on the node\'s certificate.', type=str) +@click.option('--ssl_assert_fingerprint', help='SHA-256 fingerprint of the node\'s certificate. If this value is given then root-of-trust verification isn\'t done and only the node\'s certificate fingerprint is verified.', type=str) +@click.option('--ssl_version', help='Minimum acceptable TLS/SSL version', type=str) +@click.option('--master-only', help='Only run if the single host provided is the elected master', is_flag=True, default=None) +@click.option('--skip_version_test', help='Do not check the host version', is_flag=True, default=None) @click.option('--dry-run', is_flag=True, help='Do not perform any changes.') +@click.option('--loglevel', help='Log level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])) +@click.option('--logfile', help='log file') +@click.option('--logformat', help='Log output format', type=click.Choice(['default', 'logstash', 'json', 'ecs'])) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(version=__version__) -def cli(config, dry_run, action_file): +@click.pass_context +def cli( + ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth, + opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, + ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, + dry_run, loglevel, logfile, logformat, action_file +): """ Curator for Elasticsearch indices. See http://elastic.co/guide/en/elasticsearch/client/curator/current """ - run(config, action_file, dry_run) + client_args = ClientArgs() + other_args = OtherArgs() + if config: + from_yaml = get_yaml(config) + raw_config = check_config(from_yaml) + client_args.update_settings(raw_config['client']) + other_args.update_settings(raw_config['other_settings']) + + set_logging(check_logging_config( + {'logging': override_logging(from_yaml, loglevel, logfile, logformat)})) + + hostslist = cli_hostslist(hosts) + + cli_client = prune_nones({ + 'hosts': hostslist, + 'cloud_id': cloud_id, + 'bearer_auth': bearer_auth, + 'opaque_id': opaque_id, + 'request_timeout': request_timeout, + 'http_compress': http_compress, + 'verify_certs': verify_certs, + 'ca_certs': ca_certs, + 'client_cert': client_cert, + 'client_key': client_key, + 'ssl_assert_hostname': ssl_assert_hostname, + 'ssl_assert_fingerprint': ssl_assert_fingerprint, + 'ssl_version': ssl_version + }) + + cli_other = prune_nones({ + 'master_only': master_only, + 'skip_version_test': skip_version_test, + 'username': username, + 'password': password, + 'api_key': { + 'id': id, + 'api_key': api_key + } + }) + # Remove `api_key` root key if `id` and `api_key` are both None + if id is None and api_key is None: + del cli_other['api_key'] + + # If hosts are in the config file, but cloud_id is specified at the command-line, + # we need to remove the hosts parameter as cloud_id and hosts are mutually exclusive + if cloud_id: + click.echo('cloud_id provided at CLI, superseding any other configured hosts') + client_args.hosts = None + cli_client.pop('hosts', None) + + # Likewise, if hosts are provided at the command-line, but cloud_id was in the config file, + # we need to remove the cloud_id parameter from the config file-based dictionary before merging + if hosts: + click.echo('hosts specified manually, superseding any other cloud_id or hosts') + client_args.hosts = None + client_args.cloud_id = None + cli_client.pop('cloud_id', None) + + # Update the objects if we have settings after pruning None values + if cli_client: + client_args.update_settings(cli_client) + if cli_other: + other_args.update_settings(cli_other) + run(client_args, other_args, action_file, dry_run) \ No newline at end of file diff --git a/curator/helpers/getters.py b/curator/helpers/getters.py index c00d93a2..e7b776f3 100644 --- a/curator/helpers/getters.py +++ b/curator/helpers/getters.py @@ -2,7 +2,9 @@ # :pylint disable= import logging from elasticsearch8 import exceptions as es8exc -from curator.exceptions import CuratorException, FailedExecution, MissingArgument +from es_client.defaults import VERSION_MAX, VERSION_MIN +from es_client.builder import Builder +from curator.exceptions import ClientException, CuratorException, FailedExecution, MissingArgument def byte_size(num, suffix='B'): """ @@ -22,6 +24,46 @@ def byte_size(num, suffix='B'): num /= 1024.0 return f'{num:.1f}Y{suffix}' +def get_client( + configdict=None, configfile=None, autoconnect=False, version_min=VERSION_MIN, + version_max=VERSION_MAX): + """Get an Elasticsearch Client using :py:class:`es_client.Builder` + + Build a client out of settings from `configfile` or `configdict` + If neither `configfile` nor `configdict` is provided, empty defaults will be used. + If both are provided, `configdict` will be used, and `configfile` ignored. + + :param configdict: A configuration dictionary + :param configfile: A configuration file + :param autoconnect: Connect to client automatically + :param verion_min: Minimum acceptable version of Elasticsearch (major, minor, patch) + :param verion_max: Maximum acceptable version of Elasticsearch (major, minor, patch) + + :type configdict: dict + :type configfile: str + :type autoconnect: bool + :type version_min: tuple + :type version_max: tuple + + :returns: A client connection object + :rtype: :py:class:`~.elasticsearch.Elasticsearch` + """ + logger = logging.getLogger(__name__) + logger.info('Creating client object and testing connection') + + builder = Builder( + configdict=configdict, configfile=configfile, autoconnect=autoconnect, + version_min=version_min, version_max=version_max + ) + + try: + builder.connect() + except Exception as exc: + logger.critical('Exception encountered: %s', exc) + raise ClientException from exc + + return builder.client + def get_indices(client): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` diff --git a/curator/helpers/testers.py b/curator/helpers/testers.py index 40ba6fef..809e6e0e 100644 --- a/curator/helpers/testers.py +++ b/curator/helpers/testers.py @@ -4,13 +4,38 @@ from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import NotFoundError from es_client.helpers.utils import prune_nones -from curator.helpers.getters import get_repository +from curator.helpers.getters import get_repository, get_write_index from curator.exceptions import ConfigurationError, MissingArgument, RepositoryException from curator.defaults.settings import index_filtertypes, snapshot_actions, snapshot_filtertypes from curator.validators import SchemaCheck, actions, options from curator.validators.filter_functions import validfilters from curator.helpers.utils import report_failure +def ilm_policy_check(client, alias): + """Test if alias is associated with an ILM policy + + Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` + + :param client: A client connection object + :param alias: The alias name + + :type client: :py:class:`~.elasticsearch.Elasticsearch` + :type alias: str + :rtype: bool + """ + logger = logging.getLogger(__name__) + # alias = action_obj.options['name'] + write_index = get_write_index(client, alias) + try: + idx_settings = client.indices.get_settings(index=write_index) + if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: + # logger.info('Alias %s is associated with ILM policy.', alias) + # logger.info('Skipping action %s because allow_ilm_indices is false.', idx) + return True + except KeyError: + logger.debug('No ILM policies associated with %s', alias) + return False + def repository_exists(client, repository=None): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 5aae576d..8e5ecfeb 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,16 @@ Changelog ========= +8.0.2 (? ? ?) +------------- + +**Changes** + + * Added the same CLI flags that the singletons offers. This gives much more flexibility with + regards to passing configuration settings as command-line options, particularly for Docker. + * Re-created the ``get_client`` function. It now resides in ``curator.helpers.getters`` and will + eventually see use in the Reindex class for remote connections. + 8.0.1 (10 February 2023) ------------------------ diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index e485578b..44127d2a 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,4 +1,4 @@ -:curator_version: 8.0.1 +:curator_version: 8.0.2 :curator_major: 8 :curator_doc_tree: 8.0 :es_py_version: 8.6.1