Add inventory filter capability (#698)

* Add inventory filter capability.

* Use community.library_inventory_filtering_v1 collection.

* Bump dependency to 1.0.0.

* Mention the new dependency in the changelog.

.github/workflows/ansible-test.yml

@@ -50,6 +50,8 @@ jobs:
           coverage: ${{ github.event_name == 'schedule' && 'always' || 'never' }}
           pull-request-change-detection: 'true'
           testing-type: sanity
+          pre-test-cmd: >-
+            git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1
 
   units:
     # Ansible-test on various stable branches does not yet work well with cgroups v2.
@@ -72,9 +74,7 @@ jobs:
           - '2.13'
 
     steps:
-      - name: >-
+      - name: Perform unit testing against Ansible version ${{ matrix.ansible }}
-          Perform unit testing against
-          Ansible version ${{ matrix.ansible }}
         uses: felixfontein/ansible-test-gh-action@main
         with:
           ansible-core-github-repository-slug: ${{ contains(fromJson('["2.10", "2.11"]'), matrix.ansible) && 'felixfontein/ansible' || 'ansible/ansible' }}
@@ -82,6 +82,8 @@ jobs:
           coverage: ${{ github.event_name == 'schedule' && 'always' || 'never' }}
           pull-request-change-detection: 'true'
           testing-type: units
+          pre-test-cmd: >-
+            git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1
 
   integration:
     # Ansible-test on various stable branches does not yet work well with cgroups v2.
@@ -214,10 +216,7 @@ jobs:
             target: azp/6/
 
     steps:
-      - name: >-
+      - name: Perform integration testing against Ansible version ${{ matrix.ansible }} under Python ${{ matrix.python }}
-          Perform integration testing against
-          Ansible version ${{ matrix.ansible }}
-          under Python ${{ matrix.python }}
         uses: felixfontein/ansible-test-gh-action@main
         with:
           ansible-core-github-repository-slug: ${{ contains(fromJson('["2.10", "2.11"]'), matrix.ansible) && 'felixfontein/ansible' || 'ansible/ansible' }}
@@ -235,6 +234,8 @@ jobs:
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.crypto.git ../../community/crypto
             ;
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git ../../community/general
+            ;
+            git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git ../../community/library_inventory_filtering_v1
             ${{ matrix.extra-constraints && format('; echo ''{0}'' >> tests/utils/constraints.txt', matrix.extra-constraints) || '' }}
             ;
             cat tests/utils/constraints.txt

.github/workflows/docs-push.yml

@@ -37,6 +37,7 @@ jobs:
       init-html-short-title: Community.Docker Collection Docs
       init-extra-html-theme-options: |
         documentation_home_url=https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/branch/main/
+      extra-collections: community.library_inventory_filtering_v1
 
   publish-docs-gh-pages:
     # for now we won't run this on forks

changelogs/fragments/698-filter.yml

@@ -0,0 +1,11 @@
+major_changes:
+  - "The ``community.docker`` collection now depends on the ``community.library_inventory_filtering_v1`` collection.
+     This utility collection provides host filtering functionality for inventory plugins.
+     If you use the Ansible community package, both collections are included and you do not have to do anything special.
+     If you install the collection with ``ansible-galaxy collection install``, it will be installed automatically.
+     If you install the collection by copying the files of the collection to a place where ansible-core can find it,
+     for example by cloning the git repository, you need to make sure that you also have to install the dependency
+     if you are using the inventory plugins
+     (https://github.com/ansible-collections/community.docker/pull/698)."
+minor_changes:
+  - "inventory plugins - add ``filter`` option which allows to include and exclude hosts based on Jinja2 conditions (https://github.com/ansible-collections/community.docker/pull/698, https://github.com/ansible-collections/community.docker/issues/610)."

galaxy.yml

@@ -18,6 +18,8 @@ license:
 #license_file: COPYING
 tags:
   - docker
+dependencies:
+  community.library_inventory_filtering_v1: '>=1.0.0'
 repository: https://github.com/ansible-collections/community.docker
 documentation: https://docs.ansible.com/ansible/latest/collections/community/docker/
 homepage: https://github.com/ansible-collections/community.docker

plugins/inventory/docker_containers.py

@@ -21,6 +21,7 @@ author:
 extends_documentation_fragment:
     - ansible.builtin.constructed
     - community.docker.docker.api_documentation
+    - community.library_inventory_filtering_v1.inventory_filter
 description:
     - Reads inventories from the Docker API.
     - Uses a YAML configuration file that ends with C(docker.[yml|yaml]).
@@ -101,6 +102,9 @@ options:
               See the examples for how to do that.
         type: bool
         default: false
+
+    filters:
+        version_added: 3.5.0
 '''
 
 EXAMPLES = '''
@@ -144,6 +148,18 @@ connection_type: ssh
 compose:
   ansible_ssh_host: ansible_ssh_host | default(docker_name[1:], true)
   ansible_ssh_port: ansible_ssh_port | default(22, true)
+
+# Only consider containers which have a label 'foo', or whose name starts with 'a'
+plugin: community.docker.docker_containers
+filters:
+  # Accept all containers which have a label called 'foo'
+  - include: >-
+      "foo" in docker_config.Labels
+  # Next accept all containers whose inventory_hostname starts with 'a'
+  - include: >-
+      inventory_hostname.startswith("a")
+  # Exclude all containers that didn't match any of the above filters
+  - exclude: true
 '''
 
 import re
@@ -163,6 +179,7 @@ from ansible_collections.community.docker.plugins.plugin_utils.common_api import
 )
 
 from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException
+from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host
 
 MIN_DOCKER_API = None
 
@@ -209,6 +226,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
                 if value is not None:
                     extra_facts[var_name] = value
 
+        filters = parse_filters(self.get_option('filters'))
         for container in containers:
             id = container.get('Id')
             short_id = id[:13]
@@ -220,7 +238,6 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
                 name = short_id
                 full_name = id
 
-            self.inventory.add_host(name)
             facts = dict(
                 docker_name=name,
                 docker_short_id=short_id
@@ -238,25 +255,24 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
 
             running = state.get('Running')
 
+            groups = []
+
             # Add container to groups
             image_name = config.get('Image')
             if image_name and add_legacy_groups:
-                self.inventory.add_group('image_{0}'.format(image_name))
+                groups.append('image_{0}'.format(image_name))
-                self.inventory.add_host(name, group='image_{0}'.format(image_name))
 
             stack_name = labels.get('com.docker.stack.namespace')
             if stack_name:
                 full_facts['docker_stack'] = stack_name
                 if add_legacy_groups:
-                    self.inventory.add_group('stack_{0}'.format(stack_name))
+                    groups.append('stack_{0}'.format(stack_name))
-                    self.inventory.add_host(name, group='stack_{0}'.format(stack_name))
 
             service_name = labels.get('com.docker.swarm.service.name')
             if service_name:
                 full_facts['docker_service'] = service_name
                 if add_legacy_groups:
-                    self.inventory.add_group('service_{0}'.format(service_name))
+                    groups.append('service_{0}'.format(service_name))
-                    self.inventory.add_host(name, group='service_{0}'.format(service_name))
 
             if connection_type == 'ssh':
                 # Figure out ssh IP and Port
@@ -294,9 +310,17 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
                 fact_key = self._slugify(key)
                 full_facts[fact_key] = value
 
+            if not filter_host(self, name, full_facts, filters):
+                continue
+
             if verbose_output:
                 facts.update(full_facts)
 
+            self.inventory.add_host(name)
+            for group in groups:
+                self.inventory.add_group(group)
+                self.inventory.add_host(name, group=group)
+
             for key, value in facts.items():
                 self.inventory.set_variable(name, key, value)
 

plugins/inventory/docker_machine.py

@@ -13,7 +13,8 @@ DOCUMENTATION = '''
     requirements:
         - L(Docker Machine,https://docs.docker.com/machine/)
     extends_documentation_fragment:
-        - constructed
+        - ansible.builtin.constructed
+        - community.library_inventory_filtering_v1.inventory_filter
     description:
         - Get inventory hosts from Docker Machine.
         - Uses a YAML configuration file that ends with docker_machine.(yml|yaml).
@@ -53,6 +54,8 @@ DOCUMENTATION = '''
                   named C(docker_machine_node_attributes).
             type: bool
             default: true
+        filters:
+            version_added: 3.5.0
 '''
 
 EXAMPLES = '''
@@ -94,6 +97,8 @@ from ansible.module_utils.common.process import get_bin_path
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
 from ansible.utils.display import Display
 
+from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host
+
 import json
 import re
 import subprocess
@@ -201,6 +206,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
     def _populate(self):
         daemon_env = self.get_option('daemon_env')
+        filters = parse_filters(self.get_option('filters'))
         try:
             for self.node in self._get_machine_names():
                 self.node_attrs = self._inspect_docker_machine_host(self.node)
@@ -208,6 +214,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
                     continue
 
                 machine_name = self.node_attrs['Driver']['MachineName']
+                if not filter_host(self, machine_name, self.node_attrs, filters):
+                    continue
 
                 # query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands
                 # that could be used to set environment variables to influence a local Docker client:

plugins/inventory/docker_swarm.py

@@ -17,7 +17,8 @@ DOCUMENTATION = '''
         - python >= 2.7
         - L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0
     extends_documentation_fragment:
-        - constructed
+        - ansible.builtin.constructed
+        - community.library_inventory_filtering_v1.inventory_filter
     description:
         - Reads inventories from the Docker swarm API.
         - Uses a YAML configuration file docker_swarm.[yml|yaml].
@@ -108,6 +109,8 @@ DOCUMENTATION = '''
         include_host_uri_port:
             description: Override the detected port number included in C(ansible_host_uri).
             type: int
+        filters:
+            version_added: 3.5.0
 '''
 
 EXAMPLES = '''
@@ -157,6 +160,8 @@ from ansible_collections.community.docker.plugins.module_utils.util import updat
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
 from ansible.parsing.utils.addresses import parse_address
 
+from ansible_collections.community.library_inventory_filtering_v1.plugins.plugin_utils.inventory_filter import parse_filters, filter_host
+
 try:
     import docker
     HAS_DOCKER = True
@@ -196,6 +201,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
         self.inventory.add_group('leader')
         self.inventory.add_group('nonleaders')
 
+        filters = parse_filters(self.get_option('filters'))
+
         if self.get_option('include_host_uri'):
             if self.get_option('include_host_uri_port'):
                 host_uri_port = str(self.get_option('include_host_uri_port'))
@@ -208,6 +215,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
             self.nodes = self.client.nodes.list()
             for self.node in self.nodes:
                 self.node_attrs = self.client.nodes.get(self.node.id).attrs
+                if not filter_host(self, self.node_attrs['ID'], self.node_attrs, filters):
+                    continue
                 self.inventory.add_host(self.node_attrs['ID'])
                 self.inventory.add_host(self.node_attrs['ID'], group=self.node_attrs['Spec']['Role'])
                 self.inventory.set_variable(self.node_attrs['ID'], 'ansible_host',

tests/integration/requirements.yml

@@ -5,6 +5,7 @@
 
 collections:
 - ansible.posix
-- community.internal_test_tools
 - community.crypto
 - community.general
+- community.internal_test_tools
+- community.library_inventory_filtering_v1

tests/unit/plugins/inventory/test_docker_containers.py

@@ -9,14 +9,24 @@ __metaclass__ = type
 import pytest
 
 from ansible.inventory.data import InventoryData
+from ansible.parsing.dataloader import DataLoader
+from ansible.template import Templar
 
 from ansible_collections.community.docker.plugins.inventory.docker_containers import InventoryModule
+from ansible_collections.community.docker.tests.unit.compat.mock import create_autospec
 
 
 @pytest.fixture(scope="module")
-def inventory():
+def templar():
+    dataloader = create_autospec(DataLoader, instance=True)
+    return Templar(loader=dataloader)
+
+
+@pytest.fixture(scope="module")
+def inventory(templar):
     r = InventoryModule()
     r.inventory = InventoryData()
+    r.templar = templar
     return r
 
 
@@ -114,6 +124,7 @@ def test_populate(inventory, mocker):
         'compose': {},
         'groups': {},
         'keyed_groups': {},
+        'filters': None,
     }))
     inventory._populate(client)
 
@@ -145,6 +156,7 @@ def test_populate_service(inventory, mocker):
         'groups': {},
         'keyed_groups': {},
         'docker_host': 'unix://var/run/docker.sock',
+        'filters': None,
     }))
     inventory._populate(client)
 
@@ -186,6 +198,7 @@ def test_populate_stack(inventory, mocker):
         'docker_host': 'unix://var/run/docker.sock',
         'default_ip': '127.0.0.1',
         'private_ssh_port': 22,
+        'filters': None,
     }))
     inventory._populate(client)
 
@@ -212,3 +225,46 @@ def test_populate_stack(inventory, mocker):
     assert len(inventory.inventory.groups['unix://var/run/docker.sock'].hosts) == 1
     assert len(inventory.inventory.groups) == 10
     assert len(inventory.inventory.hosts) == 1
+
+
+def test_populate_filter_none(inventory, mocker):
+    client = FakeClient(LOVING_THARP)
+
+    inventory.get_option = mocker.MagicMock(side_effect=create_get_option({
+        'verbose_output': True,
+        'connection_type': 'docker-api',
+        'add_legacy_groups': False,
+        'compose': {},
+        'groups': {},
+        'keyed_groups': {},
+        'filters': [
+            {'exclude': True},
+        ],
+    }))
+    inventory._populate(client)
+
+    assert len(inventory.inventory.hosts) == 0
+
+
+def test_populate_filter(inventory, mocker):
+    client = FakeClient(LOVING_THARP)
+
+    inventory.get_option = mocker.MagicMock(side_effect=create_get_option({
+        'verbose_output': True,
+        'connection_type': 'docker-api',
+        'add_legacy_groups': False,
+        'compose': {},
+        'groups': {},
+        'keyed_groups': {},
+        'filters': [
+            {'include': 'docker_state.Running is true'},
+            {'exclude': True},
+        ],
+    }))
+    inventory._populate(client)
+
+    host_1 = inventory.inventory.get_host('loving_tharp')
+    host_1_vars = host_1.get_vars()
+
+    assert host_1_vars['ansible_host'] == 'loving_tharp'
+    assert len(inventory.inventory.hosts) == 1

tests/unit/requirements.yml

@@ -5,3 +5,4 @@
 
 collections:
 - community.internal_test_tools
+- community.library_inventory_filtering_v1

tests/utils/shippable/shippable.sh

@@ -72,6 +72,11 @@ if [ "${test}" == "sanity/extra" ]; then
 fi
 
 # START: HACK
+
+retry git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/library_inventory_filtering_v1"
+# NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429)
+# retry ansible-galaxy -vvv collection install community.library_inventory_filtering_v1
+
 if [ "${test}" == "sanity/extra" ]; then
     # Nothing further should be added to this list.
     # This is to prevent modules or plugins in this collection having a runtime dependency on other collections.