diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index 1d7da1da..d8910d9f 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -50,6 +50,8 @@ jobs: coverage: ${{ github.event_name == 'schedule' && 'always' || 'never' }} pull-request-change-detection: 'true' testing-type: sanity + pre-test-cmd: >- + git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1 units: # Ansible-test on various stable branches does not yet work well with cgroups v2. @@ -72,9 +74,7 @@ jobs: - '2.13' steps: - - name: >- - Perform unit testing against - Ansible version ${{ matrix.ansible }} + - name: Perform unit testing against Ansible version ${{ matrix.ansible }} uses: felixfontein/ansible-test-gh-action@main with: ansible-core-github-repository-slug: ${{ contains(fromJson('["2.10", "2.11"]'), matrix.ansible) && 'felixfontein/ansible' || 'ansible/ansible' }} @@ -82,6 +82,8 @@ jobs: coverage: ${{ github.event_name == 'schedule' && 'always' || 'never' }} pull-request-change-detection: 'true' testing-type: units + pre-test-cmd: >- + git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1 integration: # Ansible-test on various stable branches does not yet work well with cgroups v2. @@ -214,10 +216,7 @@ jobs: target: azp/6/ steps: - - name: >- - Perform integration testing against - Ansible version ${{ matrix.ansible }} - under Python ${{ matrix.python }} + - name: Perform integration testing against Ansible version ${{ matrix.ansible }} under Python ${{ matrix.python }} uses: felixfontein/ansible-test-gh-action@main with: ansible-core-github-repository-slug: ${{ contains(fromJson('["2.10", "2.11"]'), matrix.ansible) && 'felixfontein/ansible' || 'ansible/ansible' }} @@ -235,6 +234,8 @@ jobs: git clone --depth=1 --single-branch https://github.com/ansible-collections/community.crypto.git ../../community/crypto ; git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git ../../community/general + ; + git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1 ${{ matrix.extra-constraints && format('; echo ''{0}'' >> tests/utils/constraints.txt', matrix.extra-constraints) || '' }} ; cat tests/utils/constraints.txt diff --git a/.github/workflows/docs-push.yml b/.github/workflows/docs-push.yml index ccc32085..8ed0d874 100644 --- a/.github/workflows/docs-push.yml +++ b/.github/workflows/docs-push.yml @@ -37,6 +37,7 @@ jobs: init-html-short-title: Community.Docker Collection Docs init-extra-html-theme-options: | documentation_home_url=https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/branch/main/ + extra-collections: community.library_inventory_filtering_v1 publish-docs-gh-pages: # for now we won't run this on forks diff --git a/changelogs/fragments/698-filter.yml b/changelogs/fragments/698-filter.yml new file mode 100644 index 00000000..31b50738 --- /dev/null +++ b/changelogs/fragments/698-filter.yml @@ -0,0 +1,11 @@ +major_changes: + - "The ``community.docker`` collection now depends on the ``community.library_inventory_filtering_v1`` collection. + This utility collection provides host filtering functionality for inventory plugins. + If you use the Ansible community package, both collections are included and you do not have to do anything special. + If you install the collection with ``ansible-galaxy collection install``, it will be installed automatically. + If you install the collection by copying the files of the collection to a place where ansible-core can find it, + for example by cloning the git repository, you need to make sure that you also have to install the dependency + if you are using the inventory plugins + (https://github.com/ansible-collections/community.docker/pull/698)." +minor_changes: + - "inventory plugins - add ``filter`` option which allows to include and exclude hosts based on Jinja2 conditions (https://github.com/ansible-collections/community.docker/pull/698, https://github.com/ansible-collections/community.docker/issues/610)." diff --git a/galaxy.yml b/galaxy.yml index 60345c04..81d520b3 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -18,6 +18,8 @@ license: #license_file: COPYING tags: - docker +dependencies: + community.library_inventory_filtering_v1: '>=1.0.0' repository: https://github.com/ansible-collections/community.docker documentation: https://docs.ansible.com/ansible/latest/collections/community/docker/ homepage: https://github.com/ansible-collections/community.docker diff --git a/plugins/inventory/docker_containers.py b/plugins/inventory/docker_containers.py index 254e1685..db4fa677 100644 --- a/plugins/inventory/docker_containers.py +++ b/plugins/inventory/docker_containers.py @@ -21,6 +21,7 @@ author: extends_documentation_fragment: - ansible.builtin.constructed - community.docker.docker.api_documentation + - community.library_inventory_filtering_v1.inventory_filter description: - Reads inventories from the Docker API. - Uses a YAML configuration file that ends with C(docker.[yml|yaml]). @@ -101,6 +102,9 @@ options: See the examples for how to do that. type: bool default: false + + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -144,6 +148,18 @@ connection_type: ssh compose: ansible_ssh_host: ansible_ssh_host | default(docker_name[1:], true) ansible_ssh_port: ansible_ssh_port | default(22, true) + +# Only consider containers which have a label 'foo', or whose name starts with 'a' +plugin: community.docker.docker_containers +filters: + # Accept all containers which have a label called 'foo' + - include: >- + "foo" in docker_config.Labels + # Next accept all containers whose inventory_hostname starts with 'a' + - include: >- + inventory_hostname.startswith("a") + # Exclude all containers that didn't match any of the above filters + - exclude: true ''' import re @@ -163,6 +179,7 @@ from ansible_collections.community.docker.plugins.plugin_utils.common_api import ) from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException +from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host MIN_DOCKER_API = None @@ -209,6 +226,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable): if value is not None: extra_facts[var_name] = value + filters = parse_filters(self.get_option('filters')) for container in containers: id = container.get('Id') short_id = id[:13] @@ -220,7 +238,6 @@ class InventoryModule(BaseInventoryPlugin, Constructable): name = short_id full_name = id - self.inventory.add_host(name) facts = dict( docker_name=name, docker_short_id=short_id @@ -238,25 +255,24 @@ class InventoryModule(BaseInventoryPlugin, Constructable): running = state.get('Running') + groups = [] + # Add container to groups image_name = config.get('Image') if image_name and add_legacy_groups: - self.inventory.add_group('image_{0}'.format(image_name)) - self.inventory.add_host(name, group='image_{0}'.format(image_name)) + groups.append('image_{0}'.format(image_name)) stack_name = labels.get('com.docker.stack.namespace') if stack_name: full_facts['docker_stack'] = stack_name if add_legacy_groups: - self.inventory.add_group('stack_{0}'.format(stack_name)) - self.inventory.add_host(name, group='stack_{0}'.format(stack_name)) + groups.append('stack_{0}'.format(stack_name)) service_name = labels.get('com.docker.swarm.service.name') if service_name: full_facts['docker_service'] = service_name if add_legacy_groups: - self.inventory.add_group('service_{0}'.format(service_name)) - self.inventory.add_host(name, group='service_{0}'.format(service_name)) + groups.append('service_{0}'.format(service_name)) if connection_type == 'ssh': # Figure out ssh IP and Port @@ -294,9 +310,17 @@ class InventoryModule(BaseInventoryPlugin, Constructable): fact_key = self._slugify(key) full_facts[fact_key] = value + if not filter_host(self, name, full_facts, filters): + continue + if verbose_output: facts.update(full_facts) + self.inventory.add_host(name) + for group in groups: + self.inventory.add_group(group) + self.inventory.add_host(name, group=group) + for key, value in facts.items(): self.inventory.set_variable(name, key, value) diff --git a/plugins/inventory/docker_machine.py b/plugins/inventory/docker_machine.py index c9c9c2c1..82c9eb7b 100644 --- a/plugins/inventory/docker_machine.py +++ b/plugins/inventory/docker_machine.py @@ -13,7 +13,8 @@ DOCUMENTATION = ''' requirements: - L(Docker Machine,https://docs.docker.com/machine/) extends_documentation_fragment: - - constructed + - ansible.builtin.constructed + - community.library_inventory_filtering_v1.inventory_filter description: - Get inventory hosts from Docker Machine. - Uses a YAML configuration file that ends with docker_machine.(yml|yaml). @@ -53,6 +54,8 @@ DOCUMENTATION = ''' named C(docker_machine_node_attributes). type: bool default: true + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -94,6 +97,8 @@ from ansible.module_utils.common.process import get_bin_path from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable from ansible.utils.display import Display +from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host + import json import re import subprocess @@ -201,6 +206,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): def _populate(self): daemon_env = self.get_option('daemon_env') + filters = parse_filters(self.get_option('filters')) try: for self.node in self._get_machine_names(): self.node_attrs = self._inspect_docker_machine_host(self.node) @@ -208,6 +214,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): continue machine_name = self.node_attrs['Driver']['MachineName'] + if not filter_host(self, machine_name, self.node_attrs, filters): + continue # query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands # that could be used to set environment variables to influence a local Docker client: diff --git a/plugins/inventory/docker_swarm.py b/plugins/inventory/docker_swarm.py index f8c34581..035a8041 100644 --- a/plugins/inventory/docker_swarm.py +++ b/plugins/inventory/docker_swarm.py @@ -17,7 +17,8 @@ DOCUMENTATION = ''' - python >= 2.7 - L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0 extends_documentation_fragment: - - constructed + - ansible.builtin.constructed + - community.library_inventory_filtering_v1.inventory_filter description: - Reads inventories from the Docker swarm API. - Uses a YAML configuration file docker_swarm.[yml|yaml]. @@ -108,6 +109,8 @@ DOCUMENTATION = ''' include_host_uri_port: description: Override the detected port number included in C(ansible_host_uri). type: int + filters: + version_added: 3.5.0 ''' EXAMPLES = ''' @@ -157,6 +160,8 @@ from ansible_collections.community.docker.plugins.module_utils.util import updat from ansible.plugins.inventory import BaseInventoryPlugin, Constructable from ansible.parsing.utils.addresses import parse_address +from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host + try: import docker HAS_DOCKER = True @@ -196,6 +201,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable): self.inventory.add_group('leader') self.inventory.add_group('nonleaders') + filters = parse_filters(self.get_option('filters')) + if self.get_option('include_host_uri'): if self.get_option('include_host_uri_port'): host_uri_port = str(self.get_option('include_host_uri_port')) @@ -208,6 +215,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable): self.nodes = self.client.nodes.list() for self.node in self.nodes: self.node_attrs = self.client.nodes.get(self.node.id).attrs + if not filter_host(self, self.node_attrs['ID'], self.node_attrs, filters): + continue self.inventory.add_host(self.node_attrs['ID']) self.inventory.add_host(self.node_attrs['ID'], group=self.node_attrs['Spec']['Role']) self.inventory.set_variable(self.node_attrs['ID'], 'ansible_host', diff --git a/tests/integration/requirements.yml b/tests/integration/requirements.yml index 7b3e38d9..b4c045f4 100644 --- a/tests/integration/requirements.yml +++ b/tests/integration/requirements.yml @@ -5,6 +5,7 @@ collections: - ansible.posix -- community.internal_test_tools - community.crypto - community.general +- community.internal_test_tools +- community.library_inventory_filtering_v1 diff --git a/tests/unit/plugins/inventory/test_docker_containers.py b/tests/unit/plugins/inventory/test_docker_containers.py index ea16c0d9..e5761247 100644 --- a/tests/unit/plugins/inventory/test_docker_containers.py +++ b/tests/unit/plugins/inventory/test_docker_containers.py @@ -9,14 +9,24 @@ __metaclass__ = type import pytest from ansible.inventory.data import InventoryData +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar from ansible_collections.community.docker.plugins.inventory.docker_containers import InventoryModule +from ansible_collections.community.docker.tests.unit.compat.mock import create_autospec @pytest.fixture(scope="module") -def inventory(): +def templar(): + dataloader = create_autospec(DataLoader, instance=True) + return Templar(loader=dataloader) + + +@pytest.fixture(scope="module") +def inventory(templar): r = InventoryModule() r.inventory = InventoryData() + r.templar = templar return r @@ -114,6 +124,7 @@ def test_populate(inventory, mocker): 'compose': {}, 'groups': {}, 'keyed_groups': {}, + 'filters': None, })) inventory._populate(client) @@ -145,6 +156,7 @@ def test_populate_service(inventory, mocker): 'groups': {}, 'keyed_groups': {}, 'docker_host': 'unix://var/run/docker.sock', + 'filters': None, })) inventory._populate(client) @@ -186,6 +198,7 @@ def test_populate_stack(inventory, mocker): 'docker_host': 'unix://var/run/docker.sock', 'default_ip': '127.0.0.1', 'private_ssh_port': 22, + 'filters': None, })) inventory._populate(client) @@ -212,3 +225,46 @@ def test_populate_stack(inventory, mocker): assert len(inventory.inventory.groups['unix://var/run/docker.sock'].hosts) == 1 assert len(inventory.inventory.groups) == 10 assert len(inventory.inventory.hosts) == 1 + + +def test_populate_filter_none(inventory, mocker): + client = FakeClient(LOVING_THARP) + + inventory.get_option = mocker.MagicMock(side_effect=create_get_option({ + 'verbose_output': True, + 'connection_type': 'docker-api', + 'add_legacy_groups': False, + 'compose': {}, + 'groups': {}, + 'keyed_groups': {}, + 'filters': [ + {'exclude': True}, + ], + })) + inventory._populate(client) + + assert len(inventory.inventory.hosts) == 0 + + +def test_populate_filter(inventory, mocker): + client = FakeClient(LOVING_THARP) + + inventory.get_option = mocker.MagicMock(side_effect=create_get_option({ + 'verbose_output': True, + 'connection_type': 'docker-api', + 'add_legacy_groups': False, + 'compose': {}, + 'groups': {}, + 'keyed_groups': {}, + 'filters': [ + {'include': 'docker_state.Running is true'}, + {'exclude': True}, + ], + })) + inventory._populate(client) + + host_1 = inventory.inventory.get_host('loving_tharp') + host_1_vars = host_1.get_vars() + + assert host_1_vars['ansible_host'] == 'loving_tharp' + assert len(inventory.inventory.hosts) == 1 diff --git a/tests/unit/requirements.yml b/tests/unit/requirements.yml index 586a6a1b..e91ea474 100644 --- a/tests/unit/requirements.yml +++ b/tests/unit/requirements.yml @@ -5,3 +5,4 @@ collections: - community.internal_test_tools +- community.library_inventory_filtering_v1 diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index 2a8830a2..c824e2ab 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -72,6 +72,11 @@ if [ "${test}" == "sanity/extra" ]; then fi # START: HACK + +retry git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/library_inventory_filtering_v1" +# NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429) +# retry ansible-galaxy -vvv collection install community.library_inventory_filtering_v1 + if [ "${test}" == "sanity/extra" ]; then # Nothing further should be added to this list. # This is to prevent modules or plugins in this collection having a runtime dependency on other collections.