From 9e8c367c47e6fbc094c68a4326b4a66fd21c4b59 Mon Sep 17 00:00:00 2001
From: Felix Fontein <felix@fontein.de>
Date: Tue, 9 Apr 2024 17:41:12 +0200
Subject: [PATCH] docker_compose_v2: allow to specify inline compose
 definitions (#832)

* Allow to specify inline compose definitions.

* Remove comma that trips Python 2.7.

* Add tests.

* Add PyYAML as EE dependency.

* Be more explicit on PyYAML.
---
 .../832-docker_compose_v2-definition.yml      |   8 +
 meta/ee-requirements.txt                      |   1 +
 plugins/doc_fragments/compose_v2.py           |  14 +-
 plugins/module_utils/compose_v2.py            |  78 +++++-
 plugins/modules/docker_compose_v2.py          |  16 +-
 plugins/modules/docker_compose_v2_pull.py     |  12 +-
 .../tasks/tests/definition.yml                | 264 ++++++++++++++++++
 7 files changed, 374 insertions(+), 19 deletions(-)
 create mode 100644 changelogs/fragments/832-docker_compose_v2-definition.yml
 create mode 100644 tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml

diff --git a/changelogs/fragments/832-docker_compose_v2-definition.yml b/changelogs/fragments/832-docker_compose_v2-definition.yml
new file mode 100644
index 00000000..cb4c7fd3
--- /dev/null
+++ b/changelogs/fragments/832-docker_compose_v2-definition.yml
@@ -0,0 +1,8 @@
+minor_changes:
+  - "docker_compose_v2* modules - allow to provide an inline definition of the compose content
+     instead of having to provide a ``project_src`` directory with the compose file written into it
+     (https://github.com/ansible-collections/community.docker/issues/829, https://github.com/ansible-collections/community.docker/pull/832)."
+  - "The EE requirements now include PyYAML, since the ``docker_compose_v2*`` modules depend on it
+     when the ``definition`` option is used. This should not have a noticable effect on generated EEs
+     since ansible-core itself depends on PyYAML as well, and ansible-builder explicitly ignores this dependency
+     (https://github.com/ansible-collections/community.docker/pull/832)."
diff --git a/meta/ee-requirements.txt b/meta/ee-requirements.txt
index de03ff0f..099dcf9a 100644
--- a/meta/ee-requirements.txt
+++ b/meta/ee-requirements.txt
@@ -6,6 +6,7 @@ docker
 urllib3
 requests
 paramiko
+pyyaml
 
 # We assume that EEs are not based on Windows, and have Python >= 3.5.
 # (ansible-builder does not support conditionals, it will simply add
diff --git a/plugins/doc_fragments/compose_v2.py b/plugins/doc_fragments/compose_v2.py
index 4e21f974..8ab6e45f 100644
--- a/plugins/doc_fragments/compose_v2.py
+++ b/plugins/doc_fragments/compose_v2.py
@@ -18,20 +18,30 @@ options:
           - Path to a directory containing a Compose file
             (C(compose.yml), C(compose.yaml), C(docker-compose.yml), or C(docker-compose.yaml)).
           - If O(files) is provided, will look for these files in this directory instead.
+          - Mutually exclusive with O(definition).
         type: path
-        required: true
     project_name:
         description:
           - Provide a project name. If not provided, the project name is taken from the basename of O(project_src).
+          - Required when O(definition) is provided.
         type: str
     files:
         description:
           - List of Compose file names relative to O(project_src) to be used instead of the main Compose file
             (C(compose.yml), C(compose.yaml), C(docker-compose.yml), or C(docker-compose.yaml)).
           - Files are loaded and merged in the order given.
+          - Mutually exclusive with O(definition).
         type: list
         elements: path
         version_added: 3.7.0
+    definition:
+        description:
+          - Compose file describing one or more services, networks and volumes.
+          - Mutually exclusive with O(project_src) and O(files).
+          - If provided, PyYAML must be available to this module, and O(project_name) must be specified.
+          - Note that a temporary directory will be created and deleted afterwards when using this option.
+        type: dict
+        version_added: 3.9.0
     env_files:
         description:
           - By default environment files are loaded from a C(.env) file located directly under the O(project_src) directory.
@@ -45,6 +55,8 @@ options:
           - Equivalent to C(docker compose --profile).
         type: list
         elements: str
+requirements:
+  - "PyYAML if O(definition) is used"
 notes:
   - |-
     The Docker compose CLI plugin has no stable output format (see for example U(https://github.com/docker/compose/issues/10872)),
diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py
index be7101a0..7120c46b 100644
--- a/plugins/module_utils/compose_v2.py
+++ b/plugins/module_utils/compose_v2.py
@@ -9,8 +9,12 @@ __metaclass__ = type
 
 import os
 import re
+import shutil
+import tempfile
+import traceback
 from collections import namedtuple
 
+from ansible.module_utils.basic import missing_required_lib
 from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.six.moves import shlex_quote
 
@@ -21,6 +25,19 @@ from ansible_collections.community.docker.plugins.module_utils._logfmt import (
     parse_line as _parse_logfmt_line,
 )
 
+try:
+    import yaml
+    try:
+        # use C version if possible for speedup
+        from yaml import CSafeDumper as _SafeDumper
+    except ImportError:
+        from yaml import SafeDumper as _SafeDumper
+    HAS_PYYAML = True
+    PYYAML_IMPORT_ERROR = None
+except ImportError:
+    HAS_PYYAML = False
+    PYYAML_IMPORT_ERROR = traceback.format_exc()
+
 
 DOCKER_COMPOSE_FILES = ('compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml')
 
@@ -484,14 +501,28 @@ def update_failed(result, events, args, stdout, stderr, rc, cli):
 
 def common_compose_argspec():
     return dict(
-        project_src=dict(type='path', required=True),
+        project_src=dict(type='path'),
         project_name=dict(type='str'),
         files=dict(type='list', elements='path'),
+        definition=dict(type='dict'),
         env_files=dict(type='list', elements='path'),
         profiles=dict(type='list', elements='str'),
     )
 
 
+def common_compose_argspec_ex():
+    return dict(
+        argspec=common_compose_argspec(),
+        mutually_exclusive=[
+            ('definition', 'project_src'),
+            ('definition', 'files')
+        ],
+        required_by={
+            'definition': ('project_name', ),
+        },
+    )
+
+
 def combine_binary_output(*outputs):
     return b'\n'.join(out for out in outputs if out)
 
@@ -505,19 +536,38 @@ class BaseComposeManager(DockerBaseClass):
         super(BaseComposeManager, self).__init__()
         self.client = client
         self.check_mode = self.client.check_mode
+        self.cleanup_dirs = set()
         parameters = self.client.module.params
 
-        self.project_src = os.path.abspath(parameters['project_src'])
+        if parameters['definition'] is not None and not HAS_PYYAML:
+            self.fail(
+                missing_required_lib('PyYAML'),
+                exception=PYYAML_IMPORT_ERROR
+            )
+
         self.project_name = parameters['project_name']
+        if parameters['definition'] is not None:
+            self.project_src = tempfile.mkdtemp(prefix='ansible')
+            self.cleanup_dirs.add(self.project_src)
+            compose_file = os.path.join(self.project_src, 'compose.yaml')
+            self.client.module.add_cleanup_file(compose_file)
+            try:
+                with open(compose_file, 'wb') as f:
+                    yaml.dump(parameters['definition'], f, encoding="utf-8", Dumper=_SafeDumper)
+            except Exception as exc:
+                self.fail("Error writing to %s - %s" % (compose_file, to_native(exc)))
+        else:
+            self.project_src = os.path.abspath(parameters['project_src'])
+
         self.files = parameters['files']
         self.env_files = parameters['env_files']
         self.profiles = parameters['profiles']
 
         compose = self.client.get_client_plugin_info('compose')
         if compose is None:
-            self.client.fail('Docker CLI {0} does not have the compose plugin installed'.format(self.client.get_cli()))
+            self.fail('Docker CLI {0} does not have the compose plugin installed'.format(self.client.get_cli()))
         if compose['Version'] == 'dev':
-            self.client.fail(
+            self.fail(
                 'Docker CLI {0} has a compose plugin installed, but it reports version "dev".'
                 ' Please use a version of the plugin that returns a proper version.'
                 .format(self.client.get_cli())
@@ -525,23 +575,27 @@ class BaseComposeManager(DockerBaseClass):
         compose_version = compose['Version'].lstrip('v')
         self.compose_version = LooseVersion(compose_version)
         if self.compose_version < LooseVersion(min_version):
-            self.client.fail('Docker CLI {cli} has the compose plugin with version {version}; need version {min_version} or later'.format(
+            self.fail('Docker CLI {cli} has the compose plugin with version {version}; need version {min_version} or later'.format(
                 cli=self.client.get_cli(),
                 version=compose_version,
                 min_version=min_version,
             ))
 
         if not os.path.isdir(self.project_src):
-            self.client.fail('"{0}" is not a directory'.format(self.project_src))
+            self.fail('"{0}" is not a directory'.format(self.project_src))
 
         if self.files:
             for file in self.files:
                 path = os.path.join(self.project_src, file)
                 if not os.path.exists(path):
-                    self.client.fail('Cannot find Compose file "{0}" relative to project directory "{1}"'.format(file, self.project_src))
+                    self.fail('Cannot find Compose file "{0}" relative to project directory "{1}"'.format(file, self.project_src))
         elif all(not os.path.exists(os.path.join(self.project_src, f)) for f in DOCKER_COMPOSE_FILES):
             filenames = ', '.join(DOCKER_COMPOSE_FILES[:-1])
-            self.client.fail('"{0}" does not contain {1}, or {2}'.format(self.project_src, filenames, DOCKER_COMPOSE_FILES[-1]))
+            self.fail('"{0}" does not contain {1}, or {2}'.format(self.project_src, filenames, DOCKER_COMPOSE_FILES[-1]))
+
+    def fail(self, msg, **kwargs):
+        self.cleanup()
+        self.client.fail(msg, **kwargs)
 
     def get_base_args(self):
         args = ['compose', '--ansi', 'never']
@@ -622,3 +676,11 @@ class BaseComposeManager(DockerBaseClass):
             for res in ('stdout', 'stderr'):
                 if result.get(res) == '':
                     result.pop(res)
+
+    def cleanup(self):
+        for dir in self.cleanup_dirs:
+            try:
+                shutil.rmtree(dir, True)
+            except Exception:
+                # shouldn't happen, but simply ignore to be on the safe side
+                pass
diff --git a/plugins/modules/docker_compose_v2.py b/plugins/modules/docker_compose_v2.py
index 29bb81ad..9972d45f 100644
--- a/plugins/modules/docker_compose_v2.py
+++ b/plugins/modules/docker_compose_v2.py
@@ -409,7 +409,7 @@ from ansible_collections.community.docker.plugins.module_utils.common_cli import
 
 from ansible_collections.community.docker.plugins.module_utils.compose_v2 import (
     BaseComposeManager,
-    common_compose_argspec,
+    common_compose_argspec_ex,
     is_failed,
 )
 
@@ -435,13 +435,13 @@ class ServicesManager(BaseComposeManager):
 
         for key, value in self.scale.items():
             if not isinstance(key, string_types):
-                self.client.fail('The key %s for `scale` is not a string' % repr(key))
+                self.fail('The key %s for `scale` is not a string' % repr(key))
             try:
                 value = check_type_int(value)
             except TypeError as exc:
-                self.client.fail('The value %s for `scale[%s]` is not an integer' % (repr(value), repr(key)))
+                self.fail('The value %s for `scale[%s]` is not an integer' % (repr(value), repr(key)))
             if value < 0:
-                self.client.fail('The value %s for `scale[%s]` is negative' % (repr(value), repr(key)))
+                self.fail('The value %s for `scale[%s]` is negative' % (repr(value), repr(key)))
             self.scale[key] = value
 
     def run(self):
@@ -620,15 +620,19 @@ def main():
         wait=dict(type='bool', default=False),
         wait_timeout=dict(type='int'),
     )
-    argument_spec.update(common_compose_argspec())
+    argspec_ex = common_compose_argspec_ex()
+    argument_spec.update(argspec_ex.pop('argspec'))
 
     client = AnsibleModuleDockerClient(
         argument_spec=argument_spec,
         supports_check_mode=True,
+        **argspec_ex
     )
 
     try:
-        result = ServicesManager(client).run()
+        manager = ServicesManager(client)
+        result = manager.run()
+        manager.cleanup()
         client.module.exit_json(**result)
     except DockerException as e:
         client.fail('An unexpected docker error occurred: {0}'.format(to_native(e)), exception=traceback.format_exc())
diff --git a/plugins/modules/docker_compose_v2_pull.py b/plugins/modules/docker_compose_v2_pull.py
index 2b1980bf..6b091f9a 100644
--- a/plugins/modules/docker_compose_v2_pull.py
+++ b/plugins/modules/docker_compose_v2_pull.py
@@ -102,7 +102,7 @@ from ansible_collections.community.docker.plugins.module_utils.common_cli import
 
 from ansible_collections.community.docker.plugins.module_utils.compose_v2 import (
     BaseComposeManager,
-    common_compose_argspec,
+    common_compose_argspec_ex,
 )
 
 from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion
@@ -117,7 +117,7 @@ class PullManager(BaseComposeManager):
 
         if self.policy != 'always' and self.compose_version < LooseVersion('2.22.0'):
             # https://github.com/docker/compose/pull/10981 - 2.22.0
-            self.client.fail('A pull policy other than always is only supported since Docker Compose 2.22.0. {0} has version {1}'.format(
+            self.fail('A pull policy other than always is only supported since Docker Compose 2.22.0. {0} has version {1}'.format(
                 self.client.get_cli(), self.compose_version))
 
     def get_pull_cmd(self, dry_run, no_start=False):
@@ -145,15 +145,19 @@ def main():
     argument_spec = dict(
         policy=dict(type='str', choices=['always', 'missing'], default='always'),
     )
-    argument_spec.update(common_compose_argspec())
+    argspec_ex = common_compose_argspec_ex()
+    argument_spec.update(argspec_ex.pop('argspec'))
 
     client = AnsibleModuleDockerClient(
         argument_spec=argument_spec,
         supports_check_mode=True,
+        **argspec_ex
     )
 
     try:
-        result = PullManager(client).run()
+        manager = PullManager(client)
+        result = manager.run()
+        manager.cleanup()
         client.module.exit_json(**result)
     except DockerException as e:
         client.fail('An unexpected docker error occurred: {0}'.format(to_native(e)), exception=traceback.format_exc())
diff --git a/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml b/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml
new file mode 100644
index 00000000..4b88edcd
--- /dev/null
+++ b/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml
@@ -0,0 +1,264 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- vars:
+    pname: "{{ name_prefix }}-definition"
+    cname: "{{ name_prefix }}-container"
+    test_service: |
+      services:
+        {{ cname }}:
+          image: "{{ docker_test_image_alpine }}"
+          command: '/bin/sh -c "sleep 10m"'
+          stop_grace_period: 1s
+    test_service_mod: |
+      services:
+        {{ cname }}:
+          image: "{{ docker_test_image_alpine }}"
+          command: '/bin/sh -c "sleep 15m"'
+          stop_grace_period: 1s
+
+  block:
+    - name: Registering container name
+      set_fact:
+        cnames: "{{ cnames + [pname ~ '-' ~ cname ~ '-1'] }}"
+        dnetworks: "{{ dnetworks + [pname ~ '_default'] }}"
+
+####################################################################
+## Present #########################################################
+####################################################################
+
+    - name: Present (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      check_mode: true
+      register: present_1_check
+
+    - name: Present
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      register: present_1
+
+    - name: Present (idempotent check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      check_mode: true
+      register: present_2_check
+
+    - name: Present (idempotent)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      register: present_2
+
+    - name: Present (changed check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: present
+      check_mode: true
+      register: present_3_check
+
+    - name: Present (changed)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: present
+      register: present_3
+
+    - assert:
+        that:
+          - present_1_check is changed
+          - present_1 is changed
+          - present_1.containers | length == 1
+          - present_1.containers[0].Name == pname ~ '-' ~ cname ~ '-1'
+          - present_1.containers[0].Image == docker_test_image_alpine
+          - present_1.images | length == 1
+          - present_1.images[0].ContainerName == pname ~ '-' ~ cname ~ '-1'
+          - present_1.images[0].Repository == (docker_test_image_alpine | split(':') | first)
+          - present_1.images[0].Tag == (docker_test_image_alpine | split(':') | last)
+          - present_2_check is not changed
+          - present_2 is not changed
+          - present_3_check is changed
+          - present_3 is changed
+
+####################################################################
+## Absent ##########################################################
+####################################################################
+
+    - name: Absent (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: absent
+      check_mode: true
+      register: absent_1_check
+
+    - name: Absent
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: absent
+      register: absent_1
+
+    - name: Absent (idempotent check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: absent
+      check_mode: true
+      register: absent_2_check
+
+    - name: Absent (idempotent)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service_mod | from_yaml }}'
+        state: absent
+      register: absent_2
+
+    - assert:
+        that:
+          - absent_1_check is changed
+          - absent_1 is changed
+          - absent_2_check is not changed
+          - absent_2 is not changed
+
+####################################################################
+## Stopping and starting ###########################################
+####################################################################
+
+    - name: Present stopped (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      check_mode: true
+      register: present_1_check
+
+    - name: Present stopped
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      register: present_1
+
+    - name: Present stopped (idempotent check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      check_mode: true
+      register: present_2_check
+
+    - name: Present stopped (idempotent)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      register: present_2
+
+    - name: Started (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      check_mode: true
+      register: present_3_check
+
+    - name: Started
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      register: present_3
+
+    - name: Started (idempotent check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      check_mode: true
+      register: present_4_check
+
+    - name: Started (idempotent)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: present
+      register: present_4
+
+    - name: Restarted (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: restarted
+      check_mode: true
+      register: present_5_check
+
+    - name: Restarted
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: restarted
+      register: present_5
+
+    - name: Stopped (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      check_mode: true
+      register: present_6_check
+
+    - name: Stopped
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: stopped
+      register: present_6
+
+    - name: Restarted (check)
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: restarted
+      check_mode: true
+      register: present_7_check
+
+    - name: Restarted
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: restarted
+      register: present_7
+
+    - name: Cleanup
+      docker_compose_v2:
+        project_name: '{{ pname }}'
+        definition: '{{ test_service | from_yaml }}'
+        state: absent
+
+    - assert:
+        that:
+          - present_1_check is changed
+          - present_1 is changed
+          - present_2_check is not changed
+          - present_2 is not changed
+          - present_3_check is changed
+          - present_3 is changed
+          - present_4_check is not changed
+          - present_4 is not changed
+          - present_5_check is changed
+          - present_5 is changed
+          - present_6_check is changed
+          - present_6 is changed
+          - present_7_check is changed
+          - present_7 is changed