docker_image(_pull), docker_container: fix compatibility with Docker 29.0.0 (#1192) (#1198)

* Add debug flag to failing task.

* Add more debug output.

* Fix pull idempotency.

* Revert "Add more debug output."

This reverts commit 64020149bf.

* Fix casing.

* Remove unreliable test.

* Add 'debug: true' to all tasks.

* Reformat.

* Fix idempotency problem for IPv6 addresses.

* Fix expose ranges handling.

* Update changelog fragment to also mention other affected modules.

(cherry picked from commit 90c4b4c543)

changelogs/fragments/1192-docker_container.yml

@@ -0,0 +1,15 @@
+bugfixes:
+  - "docker_image - fix ``source=pull`` idempotency with Docker 29.0.0 (https://github.com/ansible-collections/community.docker/pull/1192)."
+  - "docker_image_pull - fix idempotency with Docker 29.0.0 (https://github.com/ansible-collections/community.docker/pull/1192)."
+  - "docker_container - fix ``pull`` idempotency with Docker 29.0.0 (https://github.com/ansible-collections/community.docker/pull/1192)."
+  - "docker_container - fix idempotency for IPv6 addresses with Docker 29.0.0 (https://github.com/ansible-collections/community.docker/pull/1192)."
+  - "docker_container - fix handling of exposed port ranges. So far, the module used an undocumented feature of Docker that was removed from Docker 29.0.0,
+     that allowed to pass the range to the deamon and let handle it. Now the module explodes ranges into a list of all contained ports, same as the
+     Docker CLI does. For backwards compatibility with Docker < 29.0.0, it also explodes ranges returned by the API for existing containers so that
+     comparison should only indicate a difference if the ranges actually change (https://github.com/ansible-collections/community.docker/pull/1192)."
+known_issues:
+  - "docker_container - when specifying IPv6 addresses for networks, Docker since version 29 no longer returns the orignal address used
+    when adding a container to a network, but normalizes them. The module will try to normalize IP addresses for comparison,
+    but it uses the ``ipaddress`` module from the Python 3 standard library for that. When using the module with Python 2,
+    please install the `ipaddress backport for Python 2.x <https://pypi.org/project/ipaddress/>`__
+    (https://github.com/ansible-collections/community.docker/pull/1198)."

plugins/module_utils/common_api.py

@@ -420,12 +420,21 @@ class AnsibleDockerClientBase(Client):
         except Exception as exc:
             self.fail("Error inspecting image ID %s - %s" % (image_id, str(exc)))
 
+    @staticmethod
+    def _compare_images(img1, img2):
+        if img1 is None or img2 is None:
+            return img1 == img2
+        filter_keys = {"Metadata"}
+        img1_filtered = {k: v for k, v in img1.items() if k not in filter_keys}
+        img2_filtered = {k: v for k, v in img2.items() if k not in filter_keys}
+        return img1_filtered == img2_filtered
+
     def pull_image(self, name, tag="latest", platform=None):
         '''
         Pull an image
         '''
         self.log("Pulling image %s:%s" % (name, tag))
-        old_tag = self.find_image(name, tag)
+        old_image = self.find_image(name, tag)
         try:
             repository, image_tag = parse_repository_tag(name)
             registry, repo_name = auth.resolve_repository_name(repository)
@@ -459,9 +468,9 @@ class AnsibleDockerClientBase(Client):
         except Exception as exc:
             self.fail("Error pulling image %s:%s - %s" % (name, tag, str(exc)))
 
-        new_tag = self.find_image(name, tag)
+        new_image = self.find_image(name, tag)
 
-        return new_tag, old_tag == new_tag
+        return new_image, self._compare_images(old_image, new_image)
 
 
 class AnsibleDockerClient(AnsibleDockerClientBase):

plugins/module_utils/module_container/base.py

@@ -766,38 +766,43 @@ def _preprocess_ports(module, values):
                     binds[idx] = bind
         values['published_ports'] = binds
 
-    exposed = []
+    exposed = set()
     if 'exposed_ports' in values:
         for port in values['exposed_ports']:
             port = to_text(port, errors='surrogate_or_strict').strip()
             protocol = 'tcp'
-            match = re.search(r'(/.+$)', port)
+            parts = port.split("/", maxsplit=1)
-            if match:
+            if len(parts) == 2:
-                protocol = match.group(1).replace('/', '')
+                port, protocol = parts
-                port = re.sub(r'/.+$', '', port)
+            parts = port.split("-", maxsplit=1)
-            exposed.append((port, protocol))
+            if len(parts) < 2:
+                try:
+                    exposed.add((int(port), protocol))
+                except ValueError as e:
+                    module.fail_json(msg="Cannot parse port {port!r}: {e}".format(port=port, e=e))
+            else:
+                try:
+                    start_port = int(parts[0])
+                    end_port = int(parts[1])
+                    if start_port > end_port:
+                        raise ValueError(
+                            "start port must be smaller or equal to end port."
+                        )
+                except ValueError as e:
+                    module.fail_json(msg="Cannot parse port range {port!r}: {e}".format(port=port, e=e))
+                for port in range(start_port, end_port + 1):
+                    exposed.add((port, protocol))
     if 'published_ports' in values:
         # Any published port should also be exposed
         for publish_port in values['published_ports']:
-            match = False
             if isinstance(publish_port, string_types) and '/' in publish_port:
                 port, protocol = publish_port.split('/')
                 port = int(port)
             else:
                 protocol = 'tcp'
                 port = int(publish_port)
-            for exposed_port in exposed:
+            exposed.add((port, protocol))
-                if exposed_port[1] != protocol:
+    values['ports'] = sorted(exposed)
-                    continue
-                if isinstance(exposed_port[0], string_types) and '-' in exposed_port[0]:
-                    start_port, end_port = exposed_port[0].split('-')
-                    if int(start_port) <= port <= int(end_port):
-                        match = True
-                elif exposed_port[0] == port:
-                    match = True
-            if not match:
-                exposed.append((port, protocol))
-    values['ports'] = exposed
     return values
 
 

plugins/module_utils/module_container/docker_api.py

@@ -1185,10 +1185,20 @@ def _get_values_ports(module, container, api_version, options, image, host_info)
     config = container['Config']
 
     # "ExposedPorts": null returns None type & causes AttributeError - PR #5517
-    if config.get('ExposedPorts') is not None:
-        expected_exposed = [_normalize_port(p) for p in config.get('ExposedPorts', dict()).keys()]
-    else:
     expected_exposed = []
+    if config.get('ExposedPorts') is not None:
+        for port_and_protocol in config.get("ExposedPorts", {}):
+            port, protocol = _normalize_port(port_and_protocol).rsplit("/")
+            try:
+                start, end = port.split("-", 1)
+                start_port = int(start)
+                end_port = int(end)
+                for port_no in range(start_port, end_port + 1):
+                    expected_exposed.append("{port_no}/{protocol}".format(port_no=port_no, protocol=protocol))
+                continue
+            except ValueError:
+                # Either it is not a range, or a broken one - in both cases, simply add the original form
+                expected_exposed.append("{port}/{protocol}".format(port=port, protocol=protocol))
 
     return {
         'published_ports': host_config.get('PortBindings'),
@@ -1224,8 +1234,8 @@ def _get_expected_values_ports(module, client, api_version, options, image, valu
         image_ports = [_normalize_port(p) for p in image_exposed_ports]
     param_ports = []
     if 'ports' in values:
-        param_ports = [to_text(p[0], errors='surrogate_or_strict') + '/' + p[1] for p in values['ports']]
+        param_ports = ["{0}/{1}".format(to_native(p[0]), p[1]) for p in values['ports']]
-    result = list(set(image_ports + param_ports))
+    result = sorted(set(image_ports + param_ports))
     expected_values['exposed_ports'] = result
 
     if 'publish_all_ports' in values:

plugins/module_utils/module_container/module.py

@@ -16,6 +16,7 @@ from ansible_collections.community.docker.plugins.module_utils.util import (
     DockerBaseClass,
     compare_generic,
     is_image_name_id,
+    normalize_ip_address,
     sanitize_result,
 )
 
@@ -611,9 +612,9 @@ class ContainerManager(DockerBaseClass):
             else:
                 diff = False
                 network_info_ipam = network_info.get('IPAMConfig') or {}
-                if network.get('ipv4_address') and network['ipv4_address'] != network_info_ipam.get('IPv4Address'):
+                if network.get('ipv4_address') and normalize_ip_address(network['ipv4_address']) != normalize_ip_address(network_info_ipam.get('IPv4Address')):
                     diff = True
-                if network.get('ipv6_address') and network['ipv6_address'] != network_info_ipam.get('IPv6Address'):
+                if network.get('ipv6_address') and normalize_ip_address(network['ipv6_address']) != normalize_ip_address(network_info_ipam.get('IPv6Address')):
                     diff = True
                 if network.get('aliases'):
                     if not compare_generic(network['aliases'], network_info.get('Aliases'), 'allow_more_present', 'set'):

plugins/module_utils/util.py

@@ -15,6 +15,12 @@ from ansible.module_utils.common.collections import is_sequence
 from ansible_collections.community.docker.plugins.module_utils._six import string_types, urlparse
 from ansible.module_utils.common.text.converters import to_text
 
+try:
+    import ipaddress
+    HAS_IPADDRESS = True
+except ImportError:
+    HAS_IPADDRESS = False
+
 
 DEFAULT_DOCKER_HOST = 'unix:///var/run/docker.sock'
 DEFAULT_TLS = False
@@ -423,3 +429,21 @@ def omit_none_from_dict(d):
     Return a copy of the dictionary with all keys with value None omitted.
     """
     return dict((k, v) for (k, v) in d.items() if v is not None)
+
+
+def normalize_ip_address(ip_address):
+    """
+    Given an IP address as a string, normalize it so that it can be
+    used to compare IP addresses as strings.
+    """
+    if ip_address is None:
+        return None
+    if HAS_IPADDRESS:
+        try:
+            return ipaddress.ip_address(ip_address).compressed
+        except ValueError:
+            # Fallback for invalid addresses: simply return the input
+            return ip_address
+    # If we don't have ipaddress, simply give up...
+    # This mainly affects Python 2.7.
+    return ip_address

plugins/modules/docker_container.py

@@ -23,6 +23,10 @@ notes:
   - If the module needs to recreate the container, it will only use the options provided to the module to create the new container
     (except O(image)). Therefore, always specify B(all) options relevant to the container.
   - When O(restart) is set to V(true), the module will only restart the container if no config changes are detected.
+  - When specifying IPv6 addresses for networks, Docker since version 29 no longer returns the orignal address used
+    when adding a container to a network, but normalizes them. The module will try to normalize IP addresses for comparison,
+    but it uses the C(ipaddress) module from the Python 3 standard library for that. When using the module with Python 2,
+    please install the L(ipaddress backport for Python 2.x, https://pypi.org/project/ipaddress/).
 extends_documentation_fragment:
   - community.docker.docker.api_documentation
   - community.docker.attributes

tests/integration/targets/docker_container/tasks/main.yml

@@ -37,7 +37,10 @@
   register: docker_host_info
 
 # Run the tests
-- block:
+- module_defaults:
+    community.general.docker_container:
+      debug: true
+  block:
     - include_tasks: run-test.yml
       with_fileglob:
         - "tests/*.yml"

tests/integration/targets/docker_container/tasks/tests/image-ids.yml

@@ -128,6 +128,7 @@
     image: "{{ docker_test_image_digest_base }}@sha256:{{ docker_test_image_digest_v1 }}"
     name: "{{ cname }}"
     pull: true
+    debug: true
     state: present
     force_kill: true
   register: digest_3

tests/integration/targets/docker_container/tasks/tests/options.yml

@@ -3686,18 +3686,6 @@
   register: platform_5
   ignore_errors: true
 
-- name: platform (idempotency)
-  docker_container:
-    image: "{{ docker_test_image_simple_1 }}"
-    name: "{{ cname }}"
-    state: present
-    pull: true
-    platform: 386
-    force_kill: true
-    debug: true
-  register: platform_6
-  ignore_errors: true
-
 - name: cleanup
   docker_container:
     name: "{{ cname }}"
@@ -3712,7 +3700,6 @@
       - platform_3 is not changed and platform_3 is not failed
       - platform_4 is not changed and platform_4 is not failed
       - platform_5 is changed
-      - platform_6 is not changed and platform_6 is not failed
   when: docker_api_version is version('1.41', '>=')
 - assert:
     that:

tests/integration/targets/docker_container/tasks/tests/ports.yml

@@ -106,6 +106,101 @@
     force_kill: true
   register: published_ports_3
 
+- name: published_ports -- port range (same range, but listed explicitly)
+  community.docker.docker_container:
+    image: "{{ docker_test_image_alpine }}"
+    command: '/bin/sh -c "sleep 10m"'
+    name: "{{ cname }}"
+    state: started
+    exposed_ports:
+      - "9001"
+      - "9010"
+      - "9011"
+      - "9012"
+      - "9013"
+      - "9014"
+      - "9015"
+      - "9016"
+      - "9017"
+      - "9018"
+      - "9019"
+      - "9020"
+      - "9021"
+      - "9022"
+      - "9023"
+      - "9024"
+      - "9025"
+      - "9026"
+      - "9027"
+      - "9028"
+      - "9029"
+      - "9030"
+      - "9031"
+      - "9032"
+      - "9033"
+      - "9034"
+      - "9035"
+      - "9036"
+      - "9037"
+      - "9038"
+      - "9039"
+      - "9040"
+      - "9041"
+      - "9042"
+      - "9043"
+      - "9044"
+      - "9045"
+      - "9046"
+      - "9047"
+      - "9048"
+      - "9049"
+      - "9050"
+    published_ports:
+      - "9001:9001"
+      - "9020:9020"
+      - "9021:9021"
+      - "9022:9022"
+      - "9023:9023"
+      - "9024:9024"
+      - "9025:9025"
+      - "9026:9026"
+      - "9027:9027"
+      - "9028:9028"
+      - "9029:9029"
+      - "9030:9030"
+      - "9031:9031"
+      - "9032:9032"
+      - "9033:9033"
+      - "9034:9034"
+      - "9035:9035"
+      - "9036:9036"
+      - "9037:9037"
+      - "9038:9038"
+      - "9039:9039"
+      - "9040:9040"
+      - "9041:9041"
+      - "9042:9042"
+      - "9043:9043"
+      - "9044:9044"
+      - "9045:9045"
+      - "9046:9046"
+      - "9047:9047"
+      - "9048:9048"
+      - "9049:9049"
+      - "9050:9050"
+      - "9051:9051"
+      - "9052:9052"
+      - "9053:9053"
+      - "9054:9054"
+      - "9055:9055"
+      - "9056:9056"
+      - "9057:9057"
+      - "9058:9058"
+      - "9059:9059"
+      - "9060:9060"
+    force_kill: true
+  register: published_ports_4
+
 - name: cleanup
   docker_container:
     name: "{{ cname }}"
@@ -118,6 +213,7 @@
       - published_ports_1 is changed
       - published_ports_2 is not changed
       - published_ports_3 is changed
+      - published_ports_4 is not changed
 
 ####################################################################
 ## published_ports: one-element container port range ###############