From 174c0c805851a43a76015510f3a6cf497975b8d6 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 6 Dec 2025 17:48:17 +0100 Subject: [PATCH] Docker Compose 5+: improve image layer event parsing (#1219) * Remove long deprecated version fields. * Add first JSON event parsing tests. * Improve image layer event parsing for Compose 5+. * Add 'Working' to image working actions. * Add changelog fragment. * Shorten lines. * Adjust docker_compose_v2_run tests. --- changelogs/fragments/1219-compose-v2-pull.yml | 2 + plugins/module_utils/_compose_v2.py | 19 +- .../tasks/tests/pull.yml | 2 - .../tasks/tests/basic.yml | 3 +- .../plugins/module_utils/test__compose_v2.py | 206 ++++++++++++++++++ 5 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 changelogs/fragments/1219-compose-v2-pull.yml diff --git a/changelogs/fragments/1219-compose-v2-pull.yml b/changelogs/fragments/1219-compose-v2-pull.yml new file mode 100644 index 00000000..d2e7b4e6 --- /dev/null +++ b/changelogs/fragments/1219-compose-v2-pull.yml @@ -0,0 +1,2 @@ +bugfixes: + - "docker_compose_v2, docker_compose_v2_pull - adjust parsing from image pull events to changes in Docker Compose 5.0.0 (https://github.com/ansible-collections/community.docker/pull/1219)." diff --git a/plugins/module_utils/_compose_v2.py b/plugins/module_utils/_compose_v2.py index 6b61ebaf..b22c4537 100644 --- a/plugins/module_utils/_compose_v2.py +++ b/plugins/module_utils/_compose_v2.py @@ -132,7 +132,7 @@ DOCKER_PULL_PROGRESS_DONE = frozenset( "Pull complete", ) ) -DOCKER_PULL_PROGRESS_WORKING = frozenset( +DOCKER_PULL_PROGRESS_WORKING_OLD = frozenset( ( "Pulling fs layer", "Waiting", @@ -141,6 +141,7 @@ DOCKER_PULL_PROGRESS_WORKING = frozenset( "Extracting", ) ) +DOCKER_PULL_PROGRESS_WORKING = frozenset(DOCKER_PULL_PROGRESS_WORKING_OLD | {"Working"}) class ResourceType: @@ -191,7 +192,7 @@ _RE_PULL_EVENT = re.compile( ) _DOCKER_PULL_PROGRESS_WD = sorted( - DOCKER_PULL_PROGRESS_DONE | DOCKER_PULL_PROGRESS_WORKING + DOCKER_PULL_PROGRESS_DONE | DOCKER_PULL_PROGRESS_WORKING_OLD ) _RE_PULL_PROGRESS = re.compile( @@ -494,7 +495,17 @@ def parse_json_events( # {"dry-run":true,"id":"ansible-docker-test-dc713f1f-container ==> ==>","text":"naming to ansible-docker-test-dc713f1f-image"} # (The longer form happens since Docker Compose 2.39.0) continue - if isinstance(resource_id, str) and " " in resource_id: + if ( + status in ("Working", "Done") + and isinstance(line_data.get("parent_id"), str) + and line_data["parent_id"].startswith("Image ") + ): + # Compose 5.0.0+: + # {"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"} + # {"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100} + resource_type = ResourceType.IMAGE_LAYER + resource_id = line_data["parent_id"][len("Image ") :] + elif isinstance(resource_id, str) and " " in resource_id: resource_type_str, resource_id = resource_id.split(" ", 1) try: resource_type = ResourceType.from_docker_compose_event( @@ -513,7 +524,7 @@ def parse_json_events( status, text = text, status elif ( text in DOCKER_PULL_PROGRESS_DONE - or line_data.get("text") in DOCKER_PULL_PROGRESS_WORKING + or line_data.get("text") in DOCKER_PULL_PROGRESS_WORKING_OLD ): resource_type = ResourceType.IMAGE_LAYER status, text = text, status diff --git a/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml b/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml index 88a2cfea..c599f2c8 100644 --- a/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml +++ b/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml @@ -9,12 +9,10 @@ non_existing_image: does-not-exist:latest project_src: "{{ remote_tmp_dir }}/{{ pname }}" test_service_non_existing: | - version: '3' services: {{ cname }}: image: {{ non_existing_image }} test_service_simple: | - version: '3' services: {{ cname }}: image: {{ docker_test_image_simple_1 }} diff --git a/tests/integration/targets/docker_compose_v2_run/tasks/tests/basic.yml b/tests/integration/targets/docker_compose_v2_run/tasks/tests/basic.yml index 190e657f..6abeacdc 100644 --- a/tests/integration/targets/docker_compose_v2_run/tasks/tests/basic.yml +++ b/tests/integration/targets/docker_compose_v2_run/tasks/tests/basic.yml @@ -77,7 +77,8 @@ - ansible.builtin.assert: that: - result_1.rc == 0 - - result_1.stderr == "" + # Since Compose 5, unrelated output shows up in stderr... + - result_1.stderr == "" or ("Creating" in result_1.stderr and "Created" in result_1.stderr) - >- "usr" in result_1.stdout_lines and diff --git a/tests/unit/plugins/module_utils/test__compose_v2.py b/tests/unit/plugins/module_utils/test__compose_v2.py index ebc6654f..438719ad 100644 --- a/tests/unit/plugins/module_utils/test__compose_v2.py +++ b/tests/unit/plugins/module_utils/test__compose_v2.py @@ -9,6 +9,7 @@ import pytest from ansible_collections.community.docker.plugins.module_utils._compose_v2 import ( Event, parse_events, + parse_json_events, ) from .compose_v2_test_cases import EVENT_TEST_CASES @@ -384,3 +385,208 @@ def test_parse_events( assert collected_events == events assert collected_warnings == warnings + + +JSON_TEST_CASES: list[tuple[str, str, str, list[Event], list[str]]] = [ + ( + "pull-compose-2", + "2.40.3", + '{"level":"warning","msg":"/tmp/ansible.f9pcm_i3.test/ansible-docker-test-3c46cd06-pull/docker-compose.yml: the attribute `version`' + ' is obsolete, it will be ignored, please remove it to avoid potential confusion","time":"2025-12-06T13:16:30Z"}\n' + '{"id":"ansible-docker-test-3c46cd06-cont","text":"Pulling"}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Pulling fs layer"}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Downloading","status":"[\\u003e ' + ' ] 6.89kB/599.9kB","current":6890,"total":599883,"percent":1}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Download complete","percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[==\\u003e ' + ' ] 32.77kB/599.9kB","current":32768,"total":599883,"percent":5}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[============' + '======================================\\u003e] 599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Extracting","status":"[============' + '======================================\\u003e] 599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"ansible-docker-test-3c46cd06-cont","text":"Pull complete","percent":100}\n' + '{"id":"ansible-docker-test-3c46cd06-cont","text":"Pulled"}\n', + [ + Event( + "unknown", + None, + "Warning", + "/tmp/ansible.f9pcm_i3.test/ansible-docker-test-3c46cd06-pull/docker-compose.yml: the attribute `version` is obsolete," + " it will be ignored, please remove it to avoid potential confusion", + ), + Event( + "image", + "ansible-docker-test-3c46cd06-cont", + "Pulling", + None, + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Pulling fs layer", + None, + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Downloading", + "[> ] 6.89kB/599.9kB", + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Download complete", + None, + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Extracting", + "[==> ] 32.77kB/599.9kB", + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Extracting", + "[==================================================>] 599.9kB/599.9kB", + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Extracting", + "[==================================================>] 599.9kB/599.9kB", + ), + Event( + "image-layer", + "63a26ae4e8a8", + "Pull complete", + None, + ), + Event( + "image", + "ansible-docker-test-3c46cd06-cont", + "Pulled", + None, + ), + ], + [], + ), + ( + "pull-compose-5", + "5.0.0", + '{"level":"warning","msg":"/tmp/ansible.1n0q46aj.test/ansible-docker-test-b2fa9191-pull/docker-compose.yml: the attribute' + ' `version` is obsolete, it will be ignored, please remove it to avoid potential confusion","time":"2025-12-06T13:08:22Z"}\n' + '{"id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"Pulling"}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[\\u003e ' + ' ] 6.89kB/599.9kB","current":6890,"total":599883,"percent":1}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[==============' + '====================================\\u003e] 599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working"}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[==\\u003e ' + ' ] 32.77kB/599.9kB","current":32768,"total":599883,"percent":5}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[==============' + '====================================\\u003e] 599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Working","text":"[==============' + '====================================\\u003e] 599.9kB/599.9kB","current":599883,"total":599883,"percent":100}\n' + '{"id":"63a26ae4e8a8","parent_id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","percent":100}\n' + '{"id":"Image ghcr.io/ansible-collections/simple-1:tag","status":"Done","text":"Pulled"}\n', + [ + Event( + "unknown", + None, + "Warning", + "/tmp/ansible.1n0q46aj.test/ansible-docker-test-b2fa9191-pull/docker-compose.yml: the attribute `version`" + " is obsolete, it will be ignored, please remove it to avoid potential confusion", + ), + Event( + "image", + "ghcr.io/ansible-collections/simple-1:tag", + "Pulling", + "Working", + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + None, + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + "[> ] 6.89kB/599.9kB", + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + "[==================================================>] 599.9kB/599.9kB", + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + None, + ), + Event( + "image-layer", "ghcr.io/ansible-collections/simple-1:tag", "Done", None + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + "[==> ] 32.77kB/599.9kB", + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + "[==================================================>] 599.9kB/599.9kB", + ), + Event( + "image-layer", + "ghcr.io/ansible-collections/simple-1:tag", + "Working", + "[==================================================>] 599.9kB/599.9kB", + ), + Event( + "image-layer", "ghcr.io/ansible-collections/simple-1:tag", "Done", None + ), + Event( + "image", "ghcr.io/ansible-collections/simple-1:tag", "Pulled", "Done" + ), + ], + [], + ), +] + + +@pytest.mark.parametrize( + "test_id, compose_version, stderr, events, warnings", + JSON_TEST_CASES, + ids=[tc[0] for tc in JSON_TEST_CASES], +) +def test_parse_json_events( + test_id: str, + compose_version: str, + stderr: str, + events: list[Event], + warnings: list[str], +) -> None: + collected_warnings = [] + + def collect_warning(msg: str) -> None: + collected_warnings.append(msg) + + collected_events = parse_json_events( + stderr.encode("utf-8"), + warn_function=collect_warning, + ) + + print(collected_events) + print(collected_warnings) + + assert collected_events == events + assert collected_warnings == warnings