From 76b2b7c061d1ceb90826633f2aa1205df8985918 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 15 Oct 2023 14:30:33 +0200 Subject: [PATCH] Add inventory filter capability. --- plugins/doc_fragments/inventory_filter.py | 36 ++++++++ plugins/inventory/docker_containers.py | 38 ++++++-- plugins/inventory/docker_machine.py | 10 ++- plugins/inventory/docker_swarm.py | 11 ++- plugins/plugin_utils/inventory_filter.py | 86 +++++++++++++++++++ .../inventory/test_docker_containers.py | 58 ++++++++++++- 6 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 plugins/doc_fragments/inventory_filter.py create mode 100644 plugins/plugin_utils/inventory_filter.py diff --git a/plugins/doc_fragments/inventory_filter.py b/plugins/doc_fragments/inventory_filter.py new file mode 100644 index 000000000..64ce2fa5f --- /dev/null +++ b/plugins/doc_fragments/inventory_filter.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +class ModuleDocFragment(object): + + # Docker doc fragment + DOCUMENTATION = r''' +options: + filters: + description: + - A list of include/exclude filters that allows to select/deselect hosts for this inventory. + - Filters are processed sequentially until the first filter where O(filters[].exclude) or + O(filters[].include) matches is found. In case O(filters[].exclude) matches, the host is + excluded, and in case O(filters[].include) matches, the host is included. In case no filter + matches, the host is included. + type: list + elements: dict + suboptions: + exclude: + description: + - A Jinja2 condition. If it matches for a host, that host is B(excluded). + - Exactly one of O(filters[].exclude) and O(filters[].include) can be specified. + type: str + include: + description: + - A Jinja2 condition. If it matches for a host, that host is B(included). + - Exactly one of O(filters[].exclude) and O(filters[].include) can be specified. + type: str +''' diff --git a/plugins/inventory/docker_containers.py b/plugins/inventory/docker_containers.py index 955b1b2eb..023df62ed 100644 --- a/plugins/inventory/docker_containers.py +++ b/plugins/inventory/docker_containers.py @@ -21,6 +21,7 @@ extends_documentation_fragment: - ansible.builtin.constructed - community.docker.docker.api_documentation + - community.docker.inventory_filter description: - Reads inventories from the Docker API. - Uses a YAML configuration file that ends with C(docker.[yml|yaml]). @@ -101,6 +102,9 @@ See the examples for how to do that. type: bool default: false + + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -144,6 +148,18 @@ compose: ansible_ssh_host: ansible_ssh_host | default(docker_name[1:], true) ansible_ssh_port: ansible_ssh_port | default(22, true) + +# Only consider containers which have a label 'foo', or whose name starts with 'a' +plugin: community.docker.docker_containers +filters: + # Accept all containers which have a label called 'foo' + - include: >- + "foo" in docker_config.Labels + # Next accept all containers whose inventory_hostname starts with 'a' + - include: >- + inventory_hostname.startswith("a") + # Exclude all containers that didn't match any of the above filters + - exclude: true ''' import re @@ -163,6 +179,7 @@ ) from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException +from ansible_collections.community.docker.plugins.plugin_utils.inventory_filter import parse_filters, filter_host MIN_DOCKER_API = None @@ -209,6 +226,7 @@ def _populate(self, client): if value is not None: extra_facts[var_name] = value + filters = parse_filters(self.get_option('filters')) for container in containers: id = container.get('Id') short_id = id[:13] @@ -220,7 +238,6 @@ def _populate(self, client): name = short_id full_name = id - self.inventory.add_host(name) facts = dict( docker_name=name, docker_short_id=short_id @@ -238,25 +255,24 @@ def _populate(self, client): running = state.get('Running') + groups = [] + # Add container to groups image_name = config.get('Image') if image_name and add_legacy_groups: - self.inventory.add_group('image_{0}'.format(image_name)) - self.inventory.add_host(name, group='image_{0}'.format(image_name)) + groups.append('image_{0}'.format(image_name)) stack_name = labels.get('com.docker.stack.namespace') if stack_name: full_facts['docker_stack'] = stack_name if add_legacy_groups: - self.inventory.add_group('stack_{0}'.format(stack_name)) - self.inventory.add_host(name, group='stack_{0}'.format(stack_name)) + groups.append('stack_{0}'.format(stack_name)) service_name = labels.get('com.docker.swarm.service.name') if service_name: full_facts['docker_service'] = service_name if add_legacy_groups: - self.inventory.add_group('service_{0}'.format(service_name)) - self.inventory.add_host(name, group='service_{0}'.format(service_name)) + groups.append('service_{0}'.format(service_name)) if connection_type == 'ssh': # Figure out ssh IP and Port @@ -294,9 +310,17 @@ def _populate(self, client): fact_key = self._slugify(key) full_facts[fact_key] = value + if not filter_host(self, name, full_facts, filters): + continue + if verbose_output: facts.update(full_facts) + self.inventory.add_host(name) + for group in groups: + self.inventory.add_group(group) + self.inventory.add_host(name, group=group) + for key, value in facts.items(): self.inventory.set_variable(name, key, value) diff --git a/plugins/inventory/docker_machine.py b/plugins/inventory/docker_machine.py index 9edf7c27e..cea2f87b2 100644 --- a/plugins/inventory/docker_machine.py +++ b/plugins/inventory/docker_machine.py @@ -13,7 +13,8 @@ requirements: - L(Docker Machine,https://docs.docker.com/machine/) extends_documentation_fragment: - - constructed + - ansible.builtin.constructed + - community.docker.inventory_filter description: - Get inventory hosts from Docker Machine. - Uses a YAML configuration file that ends with docker_machine.(yml|yaml). @@ -53,6 +54,8 @@ named C(docker_machine_node_attributes). type: bool default: true + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -94,6 +97,8 @@ from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable from ansible.utils.display import Display +from ansible_collections.community.docker.plugins.plugin_utils.inventory_filter import parse_filters, filter_host + import json import re import subprocess @@ -201,6 +206,7 @@ def _should_skip_host(self, machine_name, env_var_tuples, daemon_env): def _populate(self): daemon_env = self.get_option('daemon_env') + filters = parse_filters(self.get_option('filters')) try: for self.node in self._get_machine_names(): self.node_attrs = self._inspect_docker_machine_host(self.node) @@ -208,6 +214,8 @@ def _populate(self): continue machine_name = self.node_attrs['Driver']['MachineName'] + if not filter_host(self, machine_name, self.node_attrs, filters): + continue # query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands # that could be used to set environment variables to influence a local Docker client: diff --git a/plugins/inventory/docker_swarm.py b/plugins/inventory/docker_swarm.py index ab2e12122..28a5c09a8 100644 --- a/plugins/inventory/docker_swarm.py +++ b/plugins/inventory/docker_swarm.py @@ -17,7 +17,8 @@ - python >= 2.7 - L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0 extends_documentation_fragment: - - constructed + - ansible.builtin.constructed + - community.docker.inventory_filter description: - Reads inventories from the Docker swarm API. - Uses a YAML configuration file docker_swarm.[yml|yaml]. @@ -106,6 +107,8 @@ include_host_uri_port: description: Override the detected port number included in C(ansible_host_uri). type: int + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -155,6 +158,8 @@ from ansible.plugins.inventory import BaseInventoryPlugin, Constructable from ansible.parsing.utils.addresses import parse_address +from ansible_collections.community.docker.plugins.plugin_utils.inventory_filter import parse_filters, filter_host + try: import docker HAS_DOCKER = True @@ -194,6 +199,8 @@ def _populate(self): self.inventory.add_group('leader') self.inventory.add_group('nonleaders') + filters = parse_filters(self.get_option('filters')) + if self.get_option('include_host_uri'): if self.get_option('include_host_uri_port'): host_uri_port = str(self.get_option('include_host_uri_port')) @@ -206,6 +213,8 @@ def _populate(self): self.nodes = self.client.nodes.list() for self.node in self.nodes: self.node_attrs = self.client.nodes.get(self.node.id).attrs + if not filter_host(self, self.node_attrs['ID'], self.node_attrs, filters): + continue self.inventory.add_host(self.node_attrs['ID']) self.inventory.add_host(self.node_attrs['ID'], group=self.node_attrs['Spec']['Role']) self.inventory.set_variable(self.node_attrs['ID'], 'ansible_host', diff --git a/plugins/plugin_utils/inventory_filter.py b/plugins/plugin_utils/inventory_filter.py new file mode 100644 index 000000000..31814d2e8 --- /dev/null +++ b/plugins/plugin_utils/inventory_filter.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils.six import string_types + + +_ALLOWED_KEYS = ('include', 'exclude') + + +def parse_filters(filters): + """ + Parse get_option('filter') and return normalized version to be fed into filter_host(). + """ + result = [] + if filters is None: + return result + for index, filter in enumerate(filters): + if len(filter) != 1: + raise AnsibleError('filter[{index}] must have exactly one key-value pair'.format( + index=index + 1, + )) + key, value = list(filter.items())[0] + if key not in _ALLOWED_KEYS: + raise AnsibleError('filter[{index}] must have a {allowed_keys} key, not "{key}"'.format( + index=index + 1, + key=key, + allowed_keys=' or '.join('"{key}"'.format(key=key) for key in _ALLOWED_KEYS), + )) + if not isinstance(value, (string_types, bool)): + raise AnsibleError('filter[{index}].{key} must be a string, not "{value_type}"'.format( + index=index + 1, + key=key, + value_type=type(value), + )) + result.append(filter) + return result + + +def filter_host(inventory_plugin, host, host_vars, filters): + """ + Determine whether a host should be accepted (``True``) or not (``False``). + """ + vars = { + 'inventory_hostname': host, + } + if host_vars: + vars.update(host_vars) + + def evaluate(condition): + if isinstance(condition, bool): + return condition + conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % condition + templar = inventory_plugin.templar + old_vars = templar.available_variables + try: + templar.available_variables = vars + return boolean(templar.template(conditional)) + except Exception as e: + raise AnsibleParserError("Could not evaluate filter condition {condition!r} for host {host}: {err}".format( + host=host, + condition=condition, + err=to_native(e), + )) + finally: + templar.available_variables = old_vars + + for filter in filters: + if 'include' in filter: + expr = filter['include'] + if evaluate(expr): + return True + if 'exclude' in filter: + expr = filter['exclude'] + if evaluate(expr): + return False + + return True diff --git a/tests/unit/plugins/inventory/test_docker_containers.py b/tests/unit/plugins/inventory/test_docker_containers.py index ea16c0d9f..e5761247d 100644 --- a/tests/unit/plugins/inventory/test_docker_containers.py +++ b/tests/unit/plugins/inventory/test_docker_containers.py @@ -9,14 +9,24 @@ import pytest from ansible.inventory.data import InventoryData +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar from ansible_collections.community.docker.plugins.inventory.docker_containers import InventoryModule +from ansible_collections.community.docker.tests.unit.compat.mock import create_autospec @pytest.fixture(scope="module") -def inventory(): +def templar(): + dataloader = create_autospec(DataLoader, instance=True) + return Templar(loader=dataloader) + + +@pytest.fixture(scope="module") +def inventory(templar): r = InventoryModule() r.inventory = InventoryData() + r.templar = templar return r @@ -114,6 +124,7 @@ def test_populate(inventory, mocker): 'compose': {}, 'groups': {}, 'keyed_groups': {}, + 'filters': None, })) inventory._populate(client) @@ -145,6 +156,7 @@ def test_populate_service(inventory, mocker): 'groups': {}, 'keyed_groups': {}, 'docker_host': 'unix://var/run/docker.sock', + 'filters': None, })) inventory._populate(client) @@ -186,6 +198,7 @@ def test_populate_stack(inventory, mocker): 'docker_host': 'unix://var/run/docker.sock', 'default_ip': '127.0.0.1', 'private_ssh_port': 22, + 'filters': None, })) inventory._populate(client) @@ -212,3 +225,46 @@ def test_populate_stack(inventory, mocker): assert len(inventory.inventory.groups['unix://var/run/docker.sock'].hosts) == 1 assert len(inventory.inventory.groups) == 10 assert len(inventory.inventory.hosts) == 1 + + +def test_populate_filter_none(inventory, mocker): + client = FakeClient(LOVING_THARP) + + inventory.get_option = mocker.MagicMock(side_effect=create_get_option({ + 'verbose_output': True, + 'connection_type': 'docker-api', + 'add_legacy_groups': False, + 'compose': {}, + 'groups': {}, + 'keyed_groups': {}, + 'filters': [ + {'exclude': True}, + ], + })) + inventory._populate(client) + + assert len(inventory.inventory.hosts) == 0 + + +def test_populate_filter(inventory, mocker): + client = FakeClient(LOVING_THARP) + + inventory.get_option = mocker.MagicMock(side_effect=create_get_option({ + 'verbose_output': True, + 'connection_type': 'docker-api', + 'add_legacy_groups': False, + 'compose': {}, + 'groups': {}, + 'keyed_groups': {}, + 'filters': [ + {'include': 'docker_state.Running is true'}, + {'exclude': True}, + ], + })) + inventory._populate(client) + + host_1 = inventory.inventory.get_host('loving_tharp') + host_1_vars = host_1.get_vars() + + assert host_1_vars['ansible_host'] == 'loving_tharp' + assert len(inventory.inventory.hosts) == 1