[stable-4] Docker Compose 5+: improve image layer event parsing (#1219) (#1220)

* Docker Compose 5+: improve image layer event parsing (#1219)

* Remove long deprecated version fields.

* Add first JSON event parsing tests.

* Improve image layer event parsing for Compose 5+.

* Add 'Working' to image working actions.

* Add changelog fragment.

* Shorten lines.

* Adjust docker_compose_v2_run tests.

(cherry picked from commit 174c0c8058)

* Remove type hints.

* Fix Python 2 compatibility when parsing JSON events.

changelogs/fragments/1219-compose-v2-pull.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - "docker_compose_v2, docker_compose_v2_pull - adjust parsing from image pull events to changes in Docker Compose 5.0.0 (https://github.com/ansible-collections/community.docker/pull/1219)."
+  - "docker_compose_v2* modules - fix Python 2 compatibility when parsing JSON events (https://github.com/ansible-collections/community.docker/pull/1220)."

plugins/module_utils/compose_v2.py

@@ -16,7 +16,7 @@ import traceback
 from collections import namedtuple
 
 from ansible.module_utils.basic import missing_required_lib
-from ansible.module_utils.common.text.converters import to_native
+from ansible.module_utils.common.text.converters import to_native, to_text
 from ansible_collections.community.docker.plugins.module_utils._six import shlex_quote, string_types
 
 from ansible_collections.community.docker.plugins.module_utils.util import DockerBaseClass
@@ -98,13 +98,16 @@ DOCKER_PULL_PROGRESS_DONE = frozenset((
     'Download complete',
     'Pull complete',
 ))
-DOCKER_PULL_PROGRESS_WORKING = frozenset((
-    'Pulling fs layer',
-    'Waiting',
-    'Downloading',
-    'Verifying Checksum',
-    'Extracting',
-))
+DOCKER_PULL_PROGRESS_WORKING_OLD = frozenset(
+    (
+        "Pulling fs layer",
+        "Waiting",
+        "Downloading",
+        "Verifying Checksum",
+        "Extracting",
+    )
+)
+DOCKER_PULL_PROGRESS_WORKING = frozenset(DOCKER_PULL_PROGRESS_WORKING_OLD | set(["Working"]))
 
 
 class ResourceType(object):
@@ -168,7 +171,7 @@ _RE_PULL_PROGRESS = re.compile(
     r'\s*'
     r'(?:|\s\[[^]]+\]\s+\S+\s*|\s+[0-9.kKmMgGbB]+/[0-9.kKmMgGbB]+\s*)'
     r'$'
-    % '|'.join(re.escape(status) for status in sorted(DOCKER_PULL_PROGRESS_DONE | DOCKER_PULL_PROGRESS_WORKING))
+    % '|'.join(re.escape(status) for status in sorted(DOCKER_PULL_PROGRESS_DONE | DOCKER_PULL_PROGRESS_WORKING_OLD))
 )
 
 _RE_ERROR_EVENT = re.compile(
@@ -392,7 +395,7 @@ def parse_json_events(stderr, warn_function=None):
                 )
             continue
         try:
-            line_data = json.loads(line)
+            line_data = json.loads(to_text(line))
         except Exception as exc:
             if warn_function:
                 warn_function(
@@ -427,9 +430,12 @@ def parse_json_events(stderr, warn_function=None):
             )
         else:
             resource_type = ResourceType.UNKNOWN
-            resource_id = line_data.get('id')
-            status = line_data.get('status')
-            text = line_data.get('text')
+            resource_id = to_native(line_data.get('id'), nonstring='passthru')
+            parent_id = to_native(line_data.get("parent_id"), nonstring='passthru')
+            status = to_native(line_data.get('status'), nonstring='passthru')
+            text = to_native(line_data.get('text'), nonstring='passthru')
+            level = to_native(line_data.get('level'), nonstring='passthru')
+            msg = to_native(line_data.get('msg'), nonstring='passthru')
             if resource_id == " " and text and text.startswith("build service "):
                 # Example:
                 # {"dry-run":true,"id":" ","text":"build service app"}
@@ -447,7 +453,13 @@ def parse_json_events(stderr, warn_function=None):
                 # {"dry-run":true,"id":"ansible-docker-test-dc713f1f-container ==> ==>","text":"naming to ansible-docker-test-dc713f1f-image"}
                 # (The longer form happens since Docker Compose 2.39.0)
                 continue
-            if isinstance(resource_id, str) and ' ' in resource_id:
+            if status in ("Working", "Done") and isinstance(parent_id, str) and parent_id.startswith("Image "):
+                # Compose 5.0.0+:
+                # {"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"}
+                # {"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100}
+                resource_type = ResourceType.IMAGE_LAYER
+                resource_id = parent_id[len("Image ") :]
+            elif isinstance(resource_id, str) and ' ' in resource_id:
                 resource_type_str, resource_id = resource_id.split(' ', 1)
                 try:
                     resource_type = ResourceType.from_docker_compose_event(resource_type_str)
@@ -463,14 +475,14 @@ def parse_json_events(stderr, warn_function=None):
             elif text in DOCKER_STATUS_PULL:
                 resource_type = ResourceType.IMAGE
                 status, text = text, status
-            elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING:
+            elif text in DOCKER_PULL_PROGRESS_DONE or text in DOCKER_PULL_PROGRESS_WORKING_OLD:
                 resource_type = ResourceType.IMAGE_LAYER
                 status, text = text, status
             elif status is None and isinstance(text, string_types) and text.startswith('Skipped - '):
                 status, text = text.split(' - ', 1)
-            elif line_data.get('level') in _JSON_LEVEL_TO_STATUS_MAP and 'msg' in line_data:
-                status = _JSON_LEVEL_TO_STATUS_MAP[line_data['level']]
-                text = line_data['msg']
+            elif level in _JSON_LEVEL_TO_STATUS_MAP and 'msg' in line_data:
+                status = _JSON_LEVEL_TO_STATUS_MAP[level]
+                text = msg
             if status not in DOCKER_STATUS_AND_WARNING and text in DOCKER_STATUS_AND_WARNING:
                 status, text = text, status
             event = Event(

tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml

@@ -9,12 +9,10 @@
     non_existing_image: does-not-exist:latest
     project_src: "{{ remote_tmp_dir }}/{{ pname }}"
     test_service_non_existing: |
-      version: '3'
       services:
         {{ cname }}:
           image: {{ non_existing_image }}
     test_service_simple: |
-      version: '3'
       services:
         {{ cname }}:
           image: {{ docker_test_image_simple_1 }}

tests/integration/targets/docker_compose_v2_run/tasks/tests/basic.yml

@@ -77,7 +77,8 @@
         - assert:
             that:
               - result_1.rc == 0
-              - result_1.stderr == ""
+              # Since Compose 5, unrelated output shows up in stderr...
+              - result_1.stderr == "" or ("Creating" in result_1.stderr and "Created" in result_1.stderr)
               - >-
                 "usr" in result_1.stdout_lines
                 and

tests/unit/plugins/module_utils/test_compose_v2.py

@@ -10,6 +10,7 @@ import pytest
 from ansible_collections.community.docker.plugins.module_utils.compose_v2 import (
     Event,
     parse_events,
+    parse_json_events,
 )
 
 from .compose_v2_test_cases import EVENT_TEST_CASES
@@ -372,3 +373,208 @@ def test_parse_events(test_id, compose_version, dry_run, nonzero_rc, stderr, eve
 
     assert collected_events == events
     assert collected_warnings == warnings
+
+
+JSON_TEST_CASES = [
+    (
+        "pull-compose-2",
+        "2.40.3",
+        '{"level":"warning","msg":"/tmp/ansible.f9pcm_i3.test/ansible-docker-test-3c46cd06-pull/docker-compose.yml: the attribute `version`'
+        ' is obsolete, it will be ignored, please remove it to avoid potential confusion","time":"2025-12-06T13:16:30Z"}\n'
+        '{"id":"ansible-docker-test-3c46cd06-cont","text":"Pulling"}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Pulling fs layer"}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Downloading","status":"[\\u003e    '
+        '                                              ]   6.89kB/599.9kB","current":6890,"total":599883,"percent":1}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Download complete","percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[==\\u003e   '
+        '                                             ]  32.77kB/599.9kB","current":32768,"total":599883,"percent":5}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[============'
+        '======================================\\u003e]  599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[============'
+        '======================================\\u003e]  599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Pull complete","percent":100}\n'
+        '{"id":"ansible-docker-test-3c46cd06-cont","text":"Pulled"}\n',
+        [
+            Event(
+                "unknown",
+                None,
+                "Warning",
+                "/tmp/ansible.f9pcm_i3.test/ansible-docker-test-3c46cd06-pull/docker-compose.yml: the attribute `version` is obsolete,"
+                " it will be ignored, please remove it to avoid potential confusion",
+            ),
+            Event(
+                "image",
+                "ansible-docker-test-3c46cd06-cont",
+                "Pulling",
+                None,
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Pulling fs layer",
+                None,
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Downloading",
+                "[>                                                  ]   6.89kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Download complete",
+                None,
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Extracting",
+                "[==>                                                ]  32.77kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Extracting",
+                "[==================================================>]  599.9kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Extracting",
+                "[==================================================>]  599.9kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "63a26ae4e8a8",
+                "Pull complete",
+                None,
+            ),
+            Event(
+                "image",
+                "ansible-docker-test-3c46cd06-cont",
+                "Pulled",
+                None,
+            ),
+        ],
+        [],
+    ),
+    (
+        "pull-compose-5",
+        "5.0.0",
+        '{"level":"warning","msg":"/tmp/ansible.1n0q46aj.test/ansible-docker-test-b2fa9191-pull/docker-compose.yml: the attribute'
+        ' `version` is obsolete, it will be ignored, please remove it to avoid potential confusion","time":"2025-12-06T13:08:22Z"}\n'
+        '{"id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"Pulling"}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[\\u003e       '
+        '                                           ]   6.89kB/599.9kB","current":6890,"total":599883,"percent":1}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[=============='
+        '====================================\\u003e]  599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[==\\u003e     '
+        '                                           ]  32.77kB/599.9kB","current":32768,"total":599883,"percent":5}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[=============='
+        '====================================\\u003e]  599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[=============='
+        '====================================\\u003e]  599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n'
+        '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100}\n'
+        '{"id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","text":"Pulled"}\n',
+        [
+            Event(
+                "unknown",
+                None,
+                "Warning",
+                "/tmp/ansible.1n0q46aj.test/ansible-docker-test-b2fa9191-pull/docker-compose.yml: the attribute `version`"
+                " is obsolete, it will be ignored, please remove it to avoid potential confusion",
+            ),
+            Event(
+                "image",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Pulling",
+                "Working",
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                None,
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                "[>                                                  ]   6.89kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                "[==================================================>]  599.9kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                None,
+            ),
+            Event(
+                "image-layer", "ghcr.io/ansible-collections/simple-1:tag", "Done", None
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                "[==>                                                ]  32.77kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                "[==================================================>]  599.9kB/599.9kB",
+            ),
+            Event(
+                "image-layer",
+                "ghcr.io/ansible-collections/simple-1:tag",
+                "Working",
+                "[==================================================>]  599.9kB/599.9kB",
+            ),
+            Event(
+                "image-layer", "ghcr.io/ansible-collections/simple-1:tag", "Done", None
+            ),
+            Event(
+                "image", "ghcr.io/ansible-collections/simple-1:tag", "Pulled", "Done"
+            ),
+        ],
+        [],
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "test_id, compose_version, stderr, events, warnings",
+    JSON_TEST_CASES,
+    ids=[tc[0] for tc in JSON_TEST_CASES],
+)
+def test_parse_json_events(
+    test_id,
+    compose_version,
+    stderr,
+    events,
+    warnings,
+):
+    collected_warnings = []
+
+    def collect_warning(msg):
+        collected_warnings.append(msg)
+
+    collected_events = parse_json_events(
+        stderr.encode("utf-8"),
+        warn_function=collect_warning,
+    )
+
+    print(collected_events)
+    print(collected_warnings)
+
+    assert collected_events == events
+    assert collected_warnings == warnings