Add docker_compose_v2_pull module (#751)

* Add docker_compose_v2_pull module.

* Improve and extend parsing of events.

* Add ignores.

* --policy is only available since Compose 2.22.0.

plugins/module_utils/compose_v2.py

@@ -57,11 +57,24 @@ DOCKER_STATUS_ERROR = frozenset((
 ))
 DOCKER_STATUS = frozenset(DOCKER_STATUS_DONE | DOCKER_STATUS_WORKING | DOCKER_STATUS_PULL | DOCKER_STATUS_ERROR)
 
+DOCKER_PULL_PROGRESS_DONE = frozenset((
+    'Already exists',
+    'Download complete',
+    'Pull complete',
+))
+DOCKER_PULL_PROGRESS_WORKING = frozenset((
+    'Pulling fs layer',
+    'Downloading',
+    'Verifying Checksum',
+    'Extracting',
+))
+
 
 class ResourceType(object):
     UNKNOWN = "unknown"
     NETWORK = "network"
     IMAGE = "image"
+    IMAGE_LAYER = "image-layer"
     VOLUME = "volume"
     CONTAINER = "container"
     SERVICE = "service"
@@ -108,6 +121,18 @@ _RE_PULL_EVENT = re.compile(
     % '|'.join(re.escape(status) for status in DOCKER_STATUS_PULL)
 )
 
+_RE_PULL_PROGRESS = re.compile(
+    r'^'
+    r'\s*'
+    r'(?P<layer>\S+)'
+    r'\s+'
+    r'(?P<status>%s)'
+    r'\s*'
+    r'(?:|\s\[[^]]+\]\s+\S+\s*)'
+    r'$'
+    % '|'.join(re.escape(status) for status in sorted(DOCKER_PULL_PROGRESS_DONE | DOCKER_PULL_PROGRESS_WORKING))
+)
+
 _RE_ERROR_EVENT = re.compile(
     r'^'
     r'\s*'
@@ -119,74 +144,150 @@ _RE_ERROR_EVENT = re.compile(
     % '|'.join(re.escape(status) for status in DOCKER_STATUS_ERROR)
 )
 
+_RE_CONTINUE_EVENT = re.compile(
+    r'^'
+    r'\s*'
+    r'(?P<resource_id>\S+)'
+    r'\s+'
+    r'-'
+    r'\s*'
+    r'(?P<msg>\S(?:|.*\S))'
+    r'$'
+)
+
+_RE_SKIPPED_EVENT = re.compile(
+    r'^'
+    r'\s*'
+    r'(?P<resource_id>\S+)'
+    r'\s+'
+    r'Skipped -'
+    r'\s*'
+    r'(?P<msg>\S(?:|.*\S))'
+    r'$'
+)
+
+
+def _extract_event(line):
+    match = _RE_RESOURCE_EVENT.match(line)
+    if match is not None:
+        status = match.group('status')
+        msg = None
+        if status not in DOCKER_STATUS:
+            status, msg = msg, status
+        return Event(
+            ResourceType.from_docker_compose_event(match.group('resource_type')),
+            match.group('resource_id'),
+            status,
+            msg,
+        )
+    match = _RE_PULL_EVENT.match(line)
+    if match:
+        return Event(
+            ResourceType.SERVICE,
+            match.group('service'),
+            match.group('status'),
+            None,
+        )
+    match = _RE_ERROR_EVENT.match(line)
+    if match:
+        return Event(
+            ResourceType.UNKNOWN,
+            match.group('resource_id'),
+            match.group('status'),
+            None,
+        )
+    match = _RE_PULL_PROGRESS.match(line)
+    if match:
+        return Event(
+            ResourceType.IMAGE_LAYER,
+            match.group('layer'),
+            match.group('status'),
+            None,
+        )
+    match = _RE_SKIPPED_EVENT.match(line)
+    if match:
+        return Event(
+            ResourceType.UNKNOWN,
+            match.group('resource_id'),
+            'Skipped',
+            match.group('msg'),
+        )
+    return None
+
+
+def _warn_missing_dry_run_prefix(line, warn_missing_dry_run_prefix, warn_function):
+    if warn_missing_dry_run_prefix and warn_function:
+        # This could be a bug, a change of docker compose's output format, ...
+        # Tell the user to report it to us :-)
+        warn_function(
+            'Event line is missing dry-run mode marker: {0!r}. Please report this at '
+            'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
+            .format(line)
+        )
+
+
+def _warn_unparsable_line(line, warn_function):
+    # This could be a bug, a change of docker compose's output format, ...
+    # Tell the user to report it to us :-)
+    if warn_function:
+        warn_function(
+            'Cannot parse event from line: {0!r}. Please report this at '
+            'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
+            .format(line)
+        )
+
+
+def _find_last_event_for(events, resource_id):
+    for index, event in enumerate(reversed(events)):
+        if event.resource_id == resource_id:
+            return len(events) - 1 - index, event
+    return None
+
+
+def _concat_event_msg(event, append_msg):
+    return Event(
+        event.resource_type,
+        event.resource_id,
+        event.status,
+        '\n'.join(msg for msg in [event.msg, append_msg] if msg is not None),
+    )
+
 
 def parse_events(stderr, dry_run=False, warn_function=None):
     events = []
     error_event = None
-    for line in stderr.splitlines():
+    stderr_lines = stderr.splitlines()
+    if stderr_lines and stderr_lines[-1] == b'':
+        del stderr_lines[-1]
+    for line in stderr_lines:
         line = to_native(line.strip())
         if not line:
             continue
+        warn_missing_dry_run_prefix = False
         if dry_run:
             if line.startswith(_DRY_RUN_MARKER):
                 line = line[len(_DRY_RUN_MARKER):].lstrip()
-            elif error_event is None and warn_function:
-                # This could be a bug, a change of docker compose's output format, ...
-                # Tell the user to report it to us :-)
-                warn_function(
-                    'Event line is missing dry-run mode marker: {0!r}. Please report this at '
-                    'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
-                    .format(line)
-                )
-        match = _RE_RESOURCE_EVENT.match(line)
-        if match is not None:
-            status = match.group('status')
-            msg = None
-            if status not in DOCKER_STATUS:
-                status, msg = msg, status
-            event = Event(
-                ResourceType.from_docker_compose_event(match.group('resource_type')),
-                match.group('resource_id'),
-                status,
-                msg,
-            )
+            else:
+                warn_missing_dry_run_prefix = True
+        event = _extract_event(line)
+        if event is not None:
             events.append(event)
-            if status in DOCKER_STATUS_ERROR:
+            if event.status in DOCKER_STATUS_ERROR:
                 error_event = event
             else:
                 error_event = None
+            _warn_missing_dry_run_prefix(line, warn_missing_dry_run_prefix, warn_function)
             continue
-        match = _RE_PULL_EVENT.match(line)
+        match = _RE_CONTINUE_EVENT.match(line)
         if match:
-            events.append(
-                Event(
-                    ResourceType.SERVICE,
-                    match.group('service'),
-                    match.group('status'),
-                    None,
-                )
-            )
-            error_event = None
-            continue
-        match = _RE_ERROR_EVENT.match(line)
-        if match:
-            error_event = Event(
-                ResourceType.UNKNOWN,
-                match.group('resource_id'),
-                match.group('status'),
-                None,
-            )
-            events.append(error_event)
-            continue
+            # Continuing an existing event
+            index_event = _find_last_event_for(events, match.group('resource_id'))
+            if index_event is not None:
+                index, event = index_event
+                events[-1] = _concat_event_msg(event, match.group('msg'))
         if error_event is not None:
             # Unparsable line that apparently belongs to the previous error event
-            error_event = Event(
-                error_event.resource_type,
-                error_event.resource_id,
-                error_event.status,
-                '\n'.join(msg for msg in [error_event.msg, line] if msg is not None),
-            )
-            events[-1] = error_event
+            events[-1] = _concat_event_msg(error_event, line)
             continue
         if line.startswith('Error '):
             # Error message that is independent of an error event
@@ -198,14 +299,18 @@ def parse_events(stderr, dry_run=False, warn_function=None):
             )
             events.append(error_event)
             continue
-        # This could be a bug, a change of docker compose's output format, ...
-        # Tell the user to report it to us :-)
-        if warn_function:
-            warn_function(
-                'Cannot parse event from line: {0!r}. Please report this at '
-                'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
-                .format(line)
+        if len(stderr_lines) == 1:
+            # **Very likely** an error message that is independent of an error event
+            error_event = Event(
+                ResourceType.UNKNOWN,
+                '',
+                'Error',
+                line,
             )
+            events.append(error_event)
+            continue
+        _warn_missing_dry_run_prefix(line, warn_missing_dry_run_prefix, warn_function)
+        _warn_unparsable_line(line, warn_function)
     return events
 
 
@@ -218,8 +323,18 @@ def has_changes(events):
 
 def extract_actions(events):
     actions = []
+    pull_actions = set()
     for event in events:
-        if event.status in DOCKER_STATUS_WORKING:
+        if event.resource_type == ResourceType.IMAGE_LAYER and event.status in DOCKER_PULL_PROGRESS_WORKING:
+            pull_id = (event.resource_id, event.status)
+            if pull_id not in pull_actions:
+                pull_actions.add(pull_id)
+                actions.append({
+                    'what': event.resource_type,
+                    'id': event.resource_id,
+                    'status': event.status,
+                })
+        if event.resource_type != ResourceType.IMAGE_LAYER and event.status in DOCKER_STATUS_WORKING:
             actions.append({
                 'what': event.resource_type,
                 'id': event.resource_id,

plugins/modules/docker_compose_v2_pull.py

@@ -0,0 +1,385 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2023, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+
+module: docker_compose_v2_pull
+
+short_description: Pull a Docker compose project
+
+version_added: 3.6.0
+
+description:
+  - Uses Docker Compose to pull images for a project.
+
+extends_documentation_fragment:
+  - community.docker.compose_v2
+  - community.docker.docker.cli_documentation
+  - community.docker.attributes
+  - community.docker.attributes.actiongroup_docker
+
+attributes:
+  check_mode:
+    support: full
+  diff_mode:
+    support: none
+
+options:
+  policy:
+    description:
+      - Whether to pull images before running. This is used when C(docker compose up) is ran.
+      - V(always) ensures that the images are always pulled, even when already present on the Docker daemon.
+      - V(missing) only pulls them when they are not present on the Docker daemon. This is only supported since Docker Compose 2.22.0.
+    type: str
+    choices:
+      - always
+      - missing
+    default: always
+
+requirements:
+  - "Docker CLI with Docker compose plugin 2.18.0 or later"
+
+author:
+  - Felix Fontein (@felixfontein)
+
+notes:
+  - |-
+    The Docker compose CLI plugin has no stable output format (see for example U(https://github.com/docker/compose/issues/10872)),
+    and for the main operations also no machine friendly output format. The module tries to accomodate this with various
+    version-dependent behavior adjustments and with testing older and newer versions of the Docker compose CLI plugin.
+
+    Currently the module is tested with multiple plugin versions between 2.18.1 and 2.23.3. The exact list of plugin versions
+    will change over time. New releases of the Docker compose CLI plugin can break this module at any time.
+
+seealso:
+  - module: community.docker.docker_compose
+'''
+
+EXAMPLES = '''
+# Examples use the django example at https://docs.docker.com/compose/django. Follow it to create the
+# flask directory
+
+- name: Run using a project directory
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Tear down existing services
+      community.docker.docker_compose_v2:
+        project_src: flask
+        state: absent
+
+    - name: Create and start services
+      community.docker.docker_compose_v2:
+        project_src: flask
+      register: output
+
+    - name: Show results
+      ansible.builtin.debug:
+        var: output
+
+    - name: Run `docker-compose up` again
+      community.docker.docker_compose_v2:
+        project_src: flask
+      register: output
+
+    - name: Show results
+      ansible.builtin.debug:
+        var: output
+
+    - ansible.builtin.assert:
+        that: not output.changed
+
+    - name: Stop all services
+      community.docker.docker_compose_v2:
+        project_src: flask
+        state: stopped
+      register: output
+
+    - name: Show results
+      ansible.builtin.debug:
+        var: output
+
+    - name: Verify that web and db services are not running
+      ansible.builtin.assert:
+        that:
+          - "not output.services.web.flask_web_1.state.running"
+          - "not output.services.db.flask_db_1.state.running"
+
+    - name: Restart services
+      community.docker.docker_compose_v2:
+        project_src: flask
+        state: restarted
+      register: output
+
+    - name: Show results
+      ansible.builtin.debug:
+        var: output
+
+    - name: Verify that web and db services are running
+      ansible.builtin.assert:
+        that:
+          - "output.services.web.flask_web_1.state.running"
+          - "output.services.db.flask_db_1.state.running"
+'''
+
+RETURN = '''
+containers:
+  description:
+    - A list of containers associated to the service.
+  returned: success
+  type: list
+  elements: dict
+  contains:
+    Command:
+      description:
+        - The container's command.
+      type: raw
+    CreatedAt:
+      description:
+        - The timestamp when the container was created.
+      type: str
+      sample: "2024-01-02 12:20:41 +0100 CET"
+    ExitCode:
+      description:
+        - The container's exit code.
+      type: int
+    Health:
+      description:
+        - The container's health check.
+      type: raw
+    ID:
+      description:
+        - The container's ID.
+      type: str
+      sample: "44a7d607219a60b7db0a4817fb3205dce46e91df2cb4b78a6100b6e27b0d3135"
+    Image:
+      description:
+        - The container's image.
+      type: str
+    Labels:
+      description:
+        - Labels for this container.
+      type: dict
+    LocalVolumes:
+      description:
+        - The local volumes count.
+      type: str
+    Mounts:
+      description:
+        - Mounts.
+      type: str
+    Name:
+      description:
+        - The container's primary name.
+      type: str
+    Names:
+      description:
+        - List of names of the container.
+      type: list
+      elements: str
+    Networks:
+      description:
+        - List of networks attached to this container.
+      type: list
+      elements: str
+    Ports:
+      description:
+        - List of port assignments as a string.
+      type: str
+    Publishers:
+      description:
+        - List of port assigments.
+      type: list
+      elements: dict
+      contains:
+        URL:
+          description:
+            - Interface the port is bound to.
+          type: str
+        TargetPort:
+          description:
+            - The container's port the published port maps to.
+          type: int
+        PublishedPort:
+          description:
+            - The port that is published.
+          type: int
+        Protocol:
+          description:
+            - The protocol.
+          type: str
+          choices:
+            - tcp
+            - udp
+    RunningFor:
+      description:
+        - Amount of time the container runs.
+      type: str
+    Service:
+      description:
+        - The name of the service.
+      type: str
+    Size:
+      description:
+        - The container's size.
+      type: str
+      sample: "0B"
+    State:
+      description:
+        - The container's state.
+      type: str
+      sample: running
+    Status:
+      description:
+        - The container's status.
+      type: str
+      sample: Up About a minute
+images:
+  description:
+    - A list of images associated to the service.
+  returned: success
+  type: list
+  elements: dict
+  contains:
+    ID:
+      description:
+        - The image's ID.
+      type: str
+      sample: sha256:c8bccc0af9571ec0d006a43acb5a8d08c4ce42b6cc7194dd6eb167976f501ef1
+    ContainerName:
+      description:
+        - Name of the conainer this image is used by.
+      type: str
+    Repository:
+      description:
+        - The repository where this image belongs to.
+      type: str
+    Tag:
+      description:
+        - The tag of the image.
+      type: str
+    Size:
+      description:
+        - The image's size in bytes.
+      type: int
+actions:
+  description:
+    - A list of actions that have been applied.
+  returned: success
+  type: list
+  elements: dict
+  contains:
+    what:
+      description:
+        - What kind of resource was changed.
+      type: str
+      sample: container
+      choices:
+        - container
+        - image
+        - network
+        - service
+        - unknown
+        - volume
+    id:
+      description:
+        - The ID of the resource that was changed.
+      type: str
+      sample: container
+    status:
+      description:
+        - The status change that happened.
+      type: str
+      sample: Creating
+      choices:
+        - Starting
+        - Exiting
+        - Restarting
+        - Creating
+        - Stopping
+        - Killing
+        - Removing
+        - Recreating
+        - Pulling
+'''
+
+import traceback
+
+from ansible.module_utils.common.text.converters import to_native
+
+from ansible_collections.community.docker.plugins.module_utils.common_cli import (
+    AnsibleModuleDockerClient,
+    DockerException,
+)
+
+from ansible_collections.community.docker.plugins.module_utils.compose_v2 import (
+    BaseComposeManager,
+    common_compose_argspec,
+)
+
+from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion
+
+
+DOCKER_COMPOSE_MINIMAL_VERSION = '2.18.0'
+
+
+class PullManager(BaseComposeManager):
+    def __init__(self, client):
+        super(PullManager, self).__init__(client, min_version=DOCKER_COMPOSE_MINIMAL_VERSION)
+        parameters = self.client.module.params
+
+        self.policy = parameters['policy']
+
+        if self.policy != 'always' and self.compose_version < LooseVersion('2.22.0'):
+            # https://github.com/docker/compose/pull/10981 - 2.22.0
+            self.client.fail('A pull policy other than always is only supported since Docker Compose 2.22.0. {0} has version {1}'.format(
+                self.client.get_cli(), self.compose_version))
+
+    def get_pull_cmd(self, dry_run, no_start=False):
+        args = self.get_base_args() + ['pull']
+        if self.policy != 'always':
+            args.extend(['--policy', self.policy])
+        if dry_run:
+            args.append('--dry-run')
+        args.append('--')
+        return args
+
+    def run(self):
+        result = dict()
+        args = self.get_pull_cmd(self.check_mode)
+        rc, stdout, stderr = self.client.call_cli(*args, cwd=self.project_src)
+        events = self.parse_events(stderr, dry_run=self.check_mode)
+        self.emit_warnings(events)
+        self.update_result(result, events, stdout, stderr)
+        self.update_failed(result, events, args, stdout, stderr, rc)
+        self.cleanup_result(result)
+        return result
+
+
+def main():
+    argument_spec = dict(
+        policy=dict(type='str', choices=['always', 'missing'], default='always'),
+    )
+    argument_spec.update(common_compose_argspec())
+
+    client = AnsibleModuleDockerClient(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    try:
+        result = PullManager(client).run()
+        client.module.exit_json(**result)
+    except DockerException as e:
+        client.fail('An unexpected docker error occurred: {0}'.format(to_native(e)), exception=traceback.format_exc())
+
+
+if __name__ == '__main__':
+    main()